offgrep 0.1.0__tar.gz → 0.2.0__tar.gz

{offgrep-0.1.0 → offgrep-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: offgrep
-Version: 0.1.0
+Version: 0.2.0
 Summary: A lousy ripgrep rip-off in pure Python
 Keywords: grep,recursive
 Author: Rainer Schwarzbach
@@ -68,4 +68,4 @@ instead of **offgrep**.
 If you – however – are in a super restricted network (eg. government)
 without a chance to install **ripgrep**,
 but are able to install Python packages, **offgrep** might be an interesting
-alterative.
+alternative.

{offgrep-0.1.0 → offgrep-0.2.0}/README.md

@@ -19,4 +19,4 @@ instead of **offgrep**.
 If you – however – are in a super restricted network (eg. government)
 without a chance to install **ripgrep**,
 but are able to install Python packages, **offgrep** might be an interesting
-alterative.
+alternative.

{offgrep-0.1.0 → offgrep-0.2.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "uv_build"
 
 [project]
 name = "offgrep"
-version = "0.1.0"
+version = "0.2.0"
 description = "A lousy ripgrep rip-off in pure Python"
 readme = "README.md"
 authors = [

{offgrep-0.1.0 → offgrep-0.2.0}/src/offgrep/__main__.py

@@ -14,7 +14,7 @@ from re import compile as re_compile
 from tomllib import loads as tomli_loads
 
 from .commons import UTF_8
-from .filesystem import IgnoredDirectoryError, iter_target_files
+from .filesystem import DEFAULT_IGNORED_DIRS, Priority, DirectoryWalker
 
 
 PACKAGE_NAME = "offgrep"
@@ -75,8 +75,10 @@ def find_in_files(
                     #
                 #
             #
-        except UnicodeDecodeError as unicode_error:
-            debug("assuming binary file %s because of %r", single_file, unicode_error)
+        except UnicodeDecodeError:
+            debug(
+                "assuming binary file %s because of unicode decode error", single_file
+            )
         except OSError as os_error:
             error(str(os_error))
             continue
@@ -102,6 +104,7 @@ def find_in_files(
 def grep_files(arguments: Namespace) -> int:
     """Search for text in files"""
     # TODO: arguments.file → read regexps from the provided file
+    default_ignore_dirs = arguments.default_ignore_dirs.split(",")
     pattern_and_or_paths = list(arguments.pattern_and_or_paths)
     try:
         pattern = pattern_and_or_paths.pop(0)
@@ -127,11 +130,12 @@ def grep_files(arguments: Namespace) -> int:
             debug("Omitting ignore rules for file %s", single_path)
             target_paths.append(single_path)
         elif single_path.is_dir():
-            try:
-                target_paths.extend(iter_target_files(single_path))
-            except IgnoredDirectoryError as id_error:
-                error(str(id_error))
-            #
+            walker = DirectoryWalker(
+                single_path,
+                default_ignore_dirs=default_ignore_dirs,
+                default_ignore_priority=arguments.default_ignore_priority,
+            )
+            target_paths.extend(walker.iter_files())
         #
     #
     find_in_files(target_paths, pattern, show_files=show_files)
@@ -140,6 +144,7 @@ def grep_files(arguments: Namespace) -> int:
 
 def list_files(arguments: Namespace) -> int:
     """List files that would be searched"""
+    default_ignore_dirs = arguments.default_ignore_dirs.split(",")
     paths = [Path(item) for item in arguments.pattern_and_or_paths]
     if not paths:
         paths.append(Path.cwd())
@@ -149,12 +154,13 @@ def list_files(arguments: Namespace) -> int:
             debug("Omitting ignore rules for file %s", single_path)
             print(single_path)
         elif single_path.is_dir():
-            try:
-                for target_path in iter_target_files(single_path):
-                    print(target_path)
-                #
-            except IgnoredDirectoryError as id_error:
-                error(str(id_error))
+            walker = DirectoryWalker(
+                single_path,
+                default_ignore_dirs=default_ignore_dirs,
+                default_ignore_priority=arguments.default_ignore_priority,
+            )
+            for target_path in walker.iter_files():
+                print(target_path)
             #
         else:
             warning("Ingored %s, neither a regular file nor a directory", single_path)
@@ -163,7 +169,10 @@ def list_files(arguments: Namespace) -> int:
     return 0
 
 
-# pylint: enable=unused-argument
+def print_supported_file_tyoes() -> int:
+    """Print the list of supported file types"""
+    print("Not implemented yet")
+    return 0
 
 
 def get_arguments() -> Namespace:
@@ -180,7 +189,11 @@ def get_arguments() -> Namespace:
     main_parser = ArgumentParser(
         prog="offgrep", description="searches text in (text) files recursively"
     )
-    main_parser.set_defaults(loglevel=WARNING)
+    main_parser.set_defaults(
+        loglevel=WARNING,
+        default_ignore_dirs=",".join(DEFAULT_IGNORED_DIRS),
+        default_ignore_priority=Priority.LOW,
+    )
     main_parser.add_argument("-V", "--version", action="version", version=version)
     main_parser.add_argument(
         "--debug",
@@ -189,11 +202,32 @@ def get_arguments() -> Namespace:
         dest="loglevel",
         help="log with debug level",
     )
+    main_parser.add_argument(
+        "--default-ignore-dirs",
+        help="Always ignore the directories named here, separated by commas."
+        " See also ---default-ignore-priority to specify when to ignore them."
+        " The preset is  %(default)r – for various version control systems.",
+    )
+    main_parser.add_argument(
+        "--default-ignore-priority",
+        choices=list(Priority),
+        type=Priority,
+        help="Priority for the ignore rules specified through -default-ignore-dirs."
+        f" Choices: %(choices)s ({str(Priority.LOW)!r} is applied first but might"
+        " be overridden by eg. directory defined rules,"
+        f" while {str(Priority.HIGH)!r} is applied last but cannot be overridden anymore)."
+        f" The preset is {str(Priority.LOW)!r}.",
+    )
     main_parser.add_argument(
         "--files",
         action="store_true",
-        help="print each file that would be searched without actually"
-        " performing the search.",
+        help="Print the path of each file that would be searched,"
+        " without actually performing the search.",
+    )
+    main_parser.add_argument(
+        "--type-list",
+        action="store_true",
+        help="Print the list of supported file types.",
     )
     main_parser.add_argument(
         "pattern_and_or_paths",
@@ -216,6 +250,10 @@ def main() -> int:
         the script rteurncode
     """
     arguments = get_arguments()
+    if arguments.type_list:
+        print_supported_file_tyoes()
+        return 0
+    #
     if arguments.files:
         return list_files(arguments)
     #

offgrep-0.2.0/src/offgrep/filesystem.py

@@ -0,0 +1,343 @@
+# -*- coding: utf-8 -*-
+
+"""
+Filesystem related functionality
+"""
+
+from collections.abc import Iterable, Iterator
+from enum import StrEnum
+from logging import debug, warning
+from pathlib import Path
+
+from .commons import EMPTY, SLASH
+from .git import get_configured_global_gitignore_path
+from .path_matching import NoRulesFound, RuleSet, iter_gitignore_rules_from_lines
+
+
+GIT_IGNORE_FILE = ".gitignore"
+MERCURIAL_IGNORE_FILE = ".hgignore"
+GENERIC_IGNORE_FILE = ".ignore"
+RIPGREP_IGNORE_FILE = ".rgignore"
+
+DEFAULT_IGNORE_FILES = (GIT_IGNORE_FILE,)
+
+IGNORE_FILES_SEQUENCE = (
+    GIT_IGNORE_FILE,
+    MERCURIAL_IGNORE_FILE,
+    GENERIC_IGNORE_FILE,
+    RIPGREP_IGNORE_FILE,
+)
+
+
+GIT_DIR = ".git"
+MERCURIAL_DIR = ".hg"
+BREEZY_DIR = ".bzr"
+DARCS_DIR = "_darcs"
+SUBVERSION_DIR = ".svn"
+
+DEFAULT_IGNORED_DIRS = (GIT_DIR, MERCURIAL_DIR, BREEZY_DIR, DARCS_DIR, SUBVERSION_DIR)
+
+IN_GIT_REPO = 0x1
+IN_MERCURIAL_REPO = 0x2
+
+_VCS_LOOKUP: dict[str, int] = {GIT_DIR: IN_GIT_REPO, MERCURIAL_DIR: IN_MERCURIAL_REPO}
+_VCS_IGNORE_FILES: dict[int, str] = {
+    IN_GIT_REPO: GIT_IGNORE_FILE,
+    IN_MERCURIAL_REPO: MERCURIAL_IGNORE_FILE,
+}
+
+
+class IgnoreFileMode(StrEnum):
+    """Ignore file handling mode"""
+
+    ALWAYS = "always"
+    """Use all ignore files, always"""
+
+    AUTOMATIC = "automatic"
+    """Use .gitignore only in Git repositories,
+    .hgignore only in Mercurial repositories,
+    and all others always
+    """
+
+    NEVER = "never"
+    """Never use any ignore files"""
+
+
+class Priority(StrEnum):
+    """Possible priorities of the default directory ruleset"""
+
+    LOW = "low"
+    HIGH = "high"
+
+
+class DirectoryWalker:
+    """Walks through a directory recursively"""
+
+    def __init__(
+        self,
+        start_directory: Path,
+        default_ignore_dirs: list[str] | tuple[str, ...] = DEFAULT_IGNORED_DIRS,
+        default_ignore_priority: Priority = Priority.LOW,
+        ignore_file_mode: IgnoreFileMode = IgnoreFileMode.AUTOMATIC,
+    ) -> None:
+        """Initialize basic values"""
+        self.start_directory = start_directory.resolve()
+        debug("Start directory was resolved to %s", self.start_directory)
+        self.default_ignore_ruleset = RuleSet(
+            title=f"Default ignored directories (Priority {default_ignore_priority})",
+            anchored_at_dir=self.start_directory,
+            rules=tuple(
+                iter_gitignore_rules_from_lines(
+                    f"{dirname.rstrip(SLASH)}{SLASH}" for dirname in default_ignore_dirs
+                )
+            ),
+        )
+        debug(str(self.default_ignore_ruleset))
+        self.default_ignore_priority = default_ignore_priority
+        self.cwd = Path.cwd()
+        self.use_relative_paths = self.start_directory.is_relative_to(self.cwd)
+        self.ignore_file_mode = ignore_file_mode
+
+    def iter_files(self) -> Iterator[Path]:
+        """Iterate over all files"""
+        vcs_bitmap, base_rulesets = self.compile_base_rulesets()
+        debug("Walking down from %s …", self.start_directory)
+        yield from self.walk_recursively(
+            self.start_directory, base_rulesets, vcs_bitmap
+        )
+
+    def compile_base_rulesets(self) -> tuple[int, tuple[RuleSet, ...]]:
+        """Compile the base rulesets"""
+        if self.ignore_file_mode == IgnoreFileMode.NEVER:
+            return 0, ()
+        #
+        vcs_bitmap = 0
+        vcs_rulesets: list[RuleSet] = [self.default_ignore_ruleset]
+        for indicator, check_for_dir, ignore_file_name in (
+            (IN_GIT_REPO, GIT_DIR, GIT_IGNORE_FILE),
+            (IN_MERCURIAL_REPO, MERCURIAL_DIR, MERCURIAL_IGNORE_FILE),
+        ):
+            additional_bitmap, additional_rulesets = self._compile_base_vcs_rulesets(
+                indicator,
+                check_for_dir=check_for_dir,
+                ignore_file_name=ignore_file_name,
+            )
+            vcs_bitmap |= additional_bitmap
+            vcs_rulesets.extend(additional_rulesets)
+        #
+        return vcs_bitmap, tuple(vcs_rulesets)
+
+    def _compile_base_vcs_rulesets(
+        self, vcs_bitmap_on_success: int, check_for_dir: str, ignore_file_name: str
+    ) -> tuple[int, tuple[RuleSet, ...]]:
+        """Compile the base VCS related rulesets"""
+        vcs_bitmap = 0
+        base_vcs_rulesets: list[RuleSet] = []
+        dirs_to_check = [self.start_directory, *self.start_directory.parents]
+        checked_dirs = []
+        for pointer in dirs_to_check:
+            debug("Checking %s for %s …", pointer, check_for_dir)
+            checked_dirs.append(pointer)
+            candidate = pointer / check_for_dir
+            if candidate.is_dir():
+                debug("… found at %s!", pointer)
+                vcs_bitmap = vcs_bitmap_on_success
+                if check_for_dir == GIT_DIR:
+                    base_vcs_rulesets.extend(
+                        compile_default_git_rulesets(
+                            repository_root=pointer,
+                            with_user_global_gitignore=True,
+                        )
+                    )
+                #
+                break
+            #
+        else:
+            debug(
+                "%s: No repository found in any parent,"
+                " continuing without (%s specific) exclusion rules",
+                check_for_dir,
+                ignore_file_name,
+            )
+            return 0, ()
+        #
+        # walk back up and handle intermediate ignore files,
+        # except for the start directory itself (that is done when walking it)
+        for idx in range(len(checked_dirs) - 1, 0, -1):
+            current_dir = checked_dirs[idx]
+            append_ruleset_from_path(
+                base_vcs_rulesets,
+                current_dir / ignore_file_name,
+                current_dir,
+            )
+            dir_to_check = checked_dirs[idx - 1]
+            is_excluded = False
+            for current_ignore_rule in base_vcs_rulesets:
+                is_excluded = current_ignore_rule.excludes_path(
+                    dir_to_check, pre_excluded=is_excluded
+                )
+            #
+            if is_excluded:
+                warning(
+                    "%s is excluded by the rulesets collected so far,"
+                    " clearing rulesets",
+                    dir_to_check,
+                )
+                return 0, ()
+            #
+        #
+        return vcs_bitmap, tuple(base_vcs_rulesets)
+
+    def _compile_local_ignore_files(
+        self,
+        directory_contents: Iterable[Path],
+        vcs_bitmap: int,
+    ) -> tuple[int, set[str]]:
+        """Compile local ignore files from the provided directory contents"""
+        match self.ignore_file_mode:
+            case IgnoreFileMode.ALWAYS:
+                return vcs_bitmap, set(IGNORE_FILES_SEQUENCE)
+            case IgnoreFileMode.NEVER:
+                return 0, set()
+            #
+        #
+        dir_names: set[str] = set()
+        file_names: set[str] = set()
+        for current_path in directory_contents:
+            item_name = current_path.name
+            if current_path.is_dir():
+                dir_names.add(item_name)
+            elif current_path.is_file():
+                file_names.add(item_name)
+        #
+        local_ignore_files = {GENERIC_IGNORE_FILE, RIPGREP_IGNORE_FILE}
+        for vcs_dir_name, indicator in _VCS_LOOKUP.items():
+            if vcs_dir_name in dir_names:
+                vcs_bitmap |= indicator
+            #
+            if vcs_bitmap & indicator:
+                local_ignore_files.add(_VCS_IGNORE_FILES[indicator])
+            #
+        #
+        return vcs_bitmap, local_ignore_files & file_names
+
+    def walk_recursively(
+        self,
+        working_directory: Path,
+        ignore_rulesets: tuple[RuleSet, ...],
+        vcs_bitmap: int,
+    ) -> Iterator[Path]:
+        """Walk through all contained files, respecting the ignore rules.
+
+        Yield paths relative to the start directory if the start directory
+        is the current working directory.
+        """
+        absolute_workdir = working_directory.resolve()
+        effective_ignore_rulesets = list(ignore_rulesets)
+        directory_contents = sorted(absolute_workdir.glob("*"), key=Path.as_posix)
+        vcs_bitmap, local_ignore_files = self._compile_local_ignore_files(
+            directory_contents, vcs_bitmap
+        )
+        for current_ignore_file in IGNORE_FILES_SEQUENCE:
+            if current_ignore_file not in local_ignore_files:
+                continue
+            #
+            append_ruleset_from_path(
+                effective_ignore_rulesets,
+                absolute_workdir / current_ignore_file,
+                absolute_workdir,
+            )
+        #
+        rulesets_to_pass: tuple[RuleSet, ...] = tuple(effective_ignore_rulesets)
+        match self.default_ignore_priority:
+            case Priority.LOW:
+                effective_ignore_rulesets.insert(0, self.default_ignore_ruleset)
+            case Priority.HIGH:
+                effective_ignore_rulesets.append(self.default_ignore_ruleset)
+            #
+        #
+        for contained_path in directory_contents:
+            # name = contained_path.name
+            if not (contained_path.is_dir() or contained_path.is_file()):
+                debug("Skipping %s, not a directory or regular file", contained_path)
+                continue
+            #
+            is_excluded = False
+            for current_ruleset in effective_ignore_rulesets:
+                is_excluded = current_ruleset.excludes_path(
+                    contained_path, pre_excluded=is_excluded
+                )
+            #
+            if is_excluded:
+                debug("Ignoring %s", contained_path)
+                continue
+            #
+            if contained_path.is_file():
+                if self.use_relative_paths:
+                    yield contained_path.relative_to(self.cwd)
+                else:
+                    yield contained_path
+                #
+            #
+            if contained_path.is_dir():
+                debug("Descending into directory %s", contained_path.name)
+                yield from self.walk_recursively(
+                    contained_path,
+                    rulesets_to_pass,
+                    vcs_bitmap,
+                )
+            #
+        #
+
+
+def append_ruleset_from_path(
+    rulesets: list[RuleSet],
+    /,
+    path: Path,
+    anchored_at_dir: Path,
+    title: str = EMPTY,
+) -> bool:
+    """load a ruleset and append it to the list"""
+    if path.is_file():
+        try:
+            if path.name == MERCURIAL_IGNORE_FILE:
+                new_ruleset = RuleSet.from_hgignore_file(
+                    path, anchored_at_dir, title=title
+                )
+            else:
+                new_ruleset = RuleSet.from_gitignore_file(
+                    path, anchored_at_dir, title=title
+                )
+            #
+        except OSError as os_error:
+            debug(str(os_error))
+            return False
+        except NoRulesFound:
+            return False
+        #
+        rulesets.append(new_ruleset)
+        return True
+    #
+    return False
+
+
+def compile_default_git_rulesets(
+    repository_root: Path,
+    with_user_global_gitignore: bool = False,
+) -> list[RuleSet]:
+    """Compile the default git rulesets"""
+    default_git_rulesets: list[RuleSet] = []
+    if with_user_global_gitignore:
+        append_ruleset_from_path(
+            default_git_rulesets,
+            path=get_configured_global_gitignore_path(),
+            anchored_at_dir=repository_root,
+            title="User global gitignore file",
+        )
+    #
+    append_ruleset_from_path(
+        default_git_rulesets,
+        path=repository_root / GIT_DIR / "info" / "exclude",
+        anchored_at_dir=repository_root,
+    )
+    return default_git_rulesets

{offgrep-0.1.0 → offgrep-0.2.0}/src/offgrep/git.py

@@ -13,9 +13,6 @@ from pathlib import Path
 GIT_COMMAND = "git"
 """The git command"""
 
-GIT_DIR = ".git"
-"""The git directory in workspaces"""
-
 USER_HOME_PATH = Path.home()
 """The user’s home directory"""
 

offgrep-0.1.0/src/offgrep/ignore.py → offgrep-0.2.0/src/offgrep/path_matching.py

@@ -4,13 +4,13 @@
 Ignore (gitignore, tba: hgignore) implementation
 """
 
+from collections.abc import Iterable, Iterator
 from dataclasses import dataclass
 from enum import StrEnum
 from functools import lru_cache
 from logging import debug, warning
-from os.path import normcase
 from pathlib import Path
-from re import compile as re_compile, escape
+from re import IGNORECASE, compile as re_compile, escape
 from typing import Self
 
 from .commons import (
@@ -50,6 +50,10 @@ Suitable for matching relative path segments.
 """
 
 
+class NoRulesFound(Exception):
+    """Raised when no rules were found in a file"""
+
+
 class Style(StrEnum):
     """Ignore style"""
 
@@ -62,30 +66,45 @@ class Style(StrEnum):
     ROOTGLOB = "rootglob"
     """Mercurial ignore entry, anchored glob style"""
 
-    REGEX = "regex"
-    """Mercurial ignore entry, regex style"""
+    REGEXP = "regexp"
+    """Mercurial ignore entry, regexp style"""
 
 
-class CompiledPattern:
-    """Pattern to match relative file paths"""
+class ExclusionRule:
+    """Rule to match relative file paths"""
 
     def __init__(
-        self, regex_pattern: str, negated: bool = False, use_normcase: bool = True
+        self, original_pattern: str, style: Style, case_sensitive: bool = True
     ) -> None:
         """compile the RE pattern
 
         Args:
-            regex_pattern: the regular expression pattern
-            negated: Flag indicating whether the pattern excludes (`False`)
-                or re-includes a previously excluded path (`True`)
-            use_normcase: Flag indicating whether to normalize the pattern
-                and the relative paths according to the operating system
-                defaults (`True`), or to always match case sensitive (`False`)
+            original_pattern: the roriginal pattern
+            style: the ignore rule style
+            case_sensitive: Flag indicating whether to use case sensitive
+                file name matching or not. Converts regex_pattern to lower case
+                if set `False`.
         """
-        self.negated = negated
-        self.regex_pattern = normcase(regex_pattern) if use_normcase else regex_pattern
-        self.use_normcase = use_normcase
-        self._cre = re_compile(regex_pattern)
+        self._negated = False
+        stripped_pattern = original_pattern.rstrip(CRLF)
+        if stripped_pattern.endswith(BACKSLASH) and not stripped_pattern[:-1].endswith(
+            BACKSLASH
+        ):
+            raise ValueError("Lines ending in a single backslash are not allowed")
+        #
+        self._original_pattern = stripped_pattern
+        effective_pattern = stripped_pattern
+        if style is Style.GITIGNORE and effective_pattern[0] == EXCLAMATION_MARK:
+            effective_pattern = effective_pattern[1:]
+            self._negated = True
+        #
+        self._style = style
+        regex_pattern = translate(effective_pattern, style=style)
+        self._cre = (
+            re_compile(regex_pattern)
+            if case_sensitive
+            else re_compile(regex_pattern, IGNORECASE)
+        )
 
     def matches(self, relative_path: str) -> bool:
         """Indicate if the pattern matches the relative path
@@ -96,110 +115,207 @@ class CompiledPattern:
         Returns:
             `True` if the path matches the pattern, `False` if not
         """
-        if self.use_normcase:
-            relative_path = normcase(relative_path)
+        if self._cre.match(relative_path):
+            debug(f"{relative_path!r} matched rule {self}")
+            return True
         #
-        return self._cre.match(relative_path) is not None
+        return False
 
-    @classmethod
-    def from_gitignore_line(
-        cls, gitignore_line: str, use_normcase: bool = True
-    ) -> Self:
-        """Factory method: compiled pattern from a `.gitignore` entry
+    def flip(self, relative_path: str, pre_excluded: bool) -> bool:
+        """Flips the exclude flag if required
+
+        Respecting the current exclusion state, matching is only performed
+        if required (no mathing of an already-excluded path against a rule that
+        could exclude it, and also no matching of an included path against a rule
+        that could re-include it).
+
+        *   pre-excluded and self._negated -> flip if the rule matches
+        *   not pre-excluded, self not negated -> flip if the rule matches
 
         Args:
-            gitignore_line: a line from a `.gitignore` file
-                (or `$GIT_DIR/info/exclude`, or the file specified by the
-                `core.excludesFile` configuration directive)
-            use_normcase: Flag indicating whether to normalize the pattern
-                and the relative paths according to the operating system
-                defaults (`True`), or to always match case sensitive (`False`)
+            relative_path: the pathc to match
+            pre_excluded: exclusion state before applying the rule
 
         Returns:
-            a new instance
-
-        Raises:
-            ValueError: if the line ends in a single backslash
+            the new exclusion state
         """
-        gitignore_line = gitignore_line.rstrip(CRLF)
-        if gitignore_line.endswith(BACKSLASH) and not gitignore_line[:-1].endswith(
-            BACKSLASH
-        ):
-            raise ValueError("Lines ending in a single backslash are not allowed")
-        #
-        _negated = False
-        if gitignore_line.startswith("!"):
-            _negated = True
-            gitignore_line = gitignore_line[1:]
+        if self._negated != pre_excluded or not self.matches(relative_path):
+            return pre_excluded
         #
-        return cls(
-            gitignore_pattern_to_full_regex(gitignore_line),
-            negated=_negated,
-            use_normcase=use_normcase,
+        return not pre_excluded
+
+    def __str__(self) -> str:
+        """String display
+
+        Returns:
+            the representation of the original pattern, with an explicit
+                abbotation if it is case INsensitive.
+        """
+        cs_display = " (CASE INSENSITIVE)" if self._cre.flags & IGNORECASE else EMPTY
+        return f"{self._original_pattern!r}{cs_display}"
+
+    def __repr__(self) -> str:
+        """String representation
+
+        Returns:
+            the Python expression that would create an equivalent instance
+        """
+        cs_display = not self._cre.flags & IGNORECASE
+        return (
+            f"self.__class__.__name__({self._original_pattern!r},"
+            f" style={self._style}, case_sensitive={cs_display})"
         )
 
 
 @dataclass(frozen=True)
-class IgnoreFile:
-    """Ignorefile data class"""
+class RuleSet:
+    """Collection of ignore rules"""
+
+    title: str
+    """Ruleset title"""
 
-    handles_directory: Path
-    """The handled directory"""
+    anchored_at_dir: Path
+    """The directory where the rules are anchored"""
 
-    rules: tuple[CompiledPattern, ...]
+    rules: tuple[ExclusionRule, ...]
     """The ignore rules set"""
 
     @classmethod
-    def from_gitignore(cls, source: Path, handles: Path) -> Self:
-        """Create an ignore file from the gitignore file"""
-        rules_collector: list[CompiledPattern] = []
-        debug("Reading gitignore file %s …", source)
+    def from_gitignore_file(
+        cls, source: Path, anchored_at_dir: Path, title: str = EMPTY
+    ) -> Self:
+        """Factory method: Create a new instance from the provided gitignore file
+
+        Args:
+            source: the gitignore file path
+            anchored_at_dir: the directory where the rules are anchored at
+            title: the title of the new ruleset
+
+        Returns:
+            the new instance
+
+        Raises:
+            NoRulesFound: if no rules were defined in the file
+        """
         with open(source, mode="rt", encoding=UTF_8) as gitignore_file:
-            for line in gitignore_file:
-                if line.startswith("#") or not line.strip():
-                    continue
-                #
-                try:
-                    rules_collector.append(CompiledPattern.from_gitignore_line(line))
-                except ValueError as value_error:
-                    warning(str(value_error))
-                #
-            #
+            defined_rules = tuple(iter_gitignore_rules_from_lines(gitignore_file))
+        #
+        if not defined_rules:
+            raise NoRulesFound
+        #
+        if not title:
+            title = str(source)
         #
-        n_rules = len(rules_collector)
-        debug("Read %s rules from %s", n_rules, source)
-        return cls(handles_directory=handles, rules=tuple(rules_collector))
+        debug("%s: defined %s rules", title, len(defined_rules))
+        return cls(title=title, anchored_at_dir=anchored_at_dir, rules=defined_rules)
+
+    @classmethod
+    def from_hgignore_file(
+        cls, source: Path, anchored_at_dir: Path, title: str = EMPTY
+    ) -> Self:
+        """Factory method: Create a new instance from the provided .hgignore file
+
+        Args:
+            source: the .hgignore file path
+            anchored_at_dir: the directory where the rules are anchored at
+            title: the title of the new ruleset
+
+        Returns:
+            the new instance
+
+        Raises:
+            NoRulesFound: if no rules were defined in the file
+        """
+        with open(source, mode="rt", encoding=UTF_8) as gitignore_file:
+            defined_rules = tuple(iter_hgignore_rules_from_lines(gitignore_file))
+        #
+        if not defined_rules:
+            raise NoRulesFound
+        #
+        if not title:
+            title = str(source)
+        #
+        debug("%s: defined %s rules", title, len(defined_rules))
+        return cls(title=title, anchored_at_dir=anchored_at_dir, rules=defined_rules)
 
     def excludes_path(self, file_or_dir: Path, pre_excluded: bool = False) -> bool:
-        """Return True if the named f file or di is excluded"""
+        """Return True if the named file or dir is excluded"""
+        # fast path if no roles are stored
+        if not self.rules:
+            return pre_excluded
+        #
         is_excluded = pre_excluded
-        relative_path = file_or_dir.relative_to(self.handles_directory)
+        relative_path = file_or_dir.relative_to(self.anchored_at_dir)
         comparable = relative_path.as_posix()
-        if relative_path.is_dir():
+        if file_or_dir.is_dir():
             comparable = f"{comparable}/"
         #
+        last_matching_rule = self.rules[0]
         for single_rule in self.rules:
-            if single_rule.matches(comparable):
-                if is_excluded and single_rule.negated:
-                    debug(
-                        "Re-including %s because of negated %s",
-                        file_or_dir,
-                        single_rule.regex_pattern,
-                    )
-                    is_excluded = False
-                else:
-                    debug(
-                        "Excluding %s because of %s",
-                        file_or_dir,
-                        single_rule.regex_pattern,
-                    )
-                    is_excluded = True
-                #
+            flipped = single_rule.flip(comparable, is_excluded)
+            if flipped ^ is_excluded:
+                last_matching_rule = single_rule
+                is_excluded = flipped
             #
         #
+        if is_excluded ^ pre_excluded:
+            new_state = "excluded" if is_excluded else "re-included"
+            debug(f"{new_state} by rule {last_matching_rule} from {self.title}")
+        #
         return is_excluded
 
 
+def iter_gitignore_rules_from_lines(lines: Iterable[str]) -> Iterator[ExclusionRule]:
+    """Create rules from the provided lines
+
+    Args:
+        lines: an Iterable of gitignore-style lines
+
+    Returns:
+        an Iterator over the created rules
+    """
+    for current_line in lines:
+        if current_line.startswith("#") or not current_line.strip():
+            continue
+        #
+        try:
+            yield ExclusionRule(current_line, style=Style.GITIGNORE)
+        except ValueError as value_error:
+            warning(str(value_error))
+        #
+    #
+
+
+def iter_hgignore_rules_from_lines(lines: Iterable[str]) -> Iterator[ExclusionRule]:
+    """Create rules from the provided lines
+
+    Args:
+        lines: an Iterable of mercurial-style lines
+
+    Returns:
+        an Iterator over the created rules
+    """
+    current_style = Style.REGEXP
+    for current_line in lines:
+        if current_line.startswith("#") or not current_line.strip():
+            continue
+        #
+        if current_line.startswith("syntax: "):
+            try:
+                current_style = Style(current_line.split()[1])
+            except ValueError as value_error:
+                warning("Invalid syntax: %s", value_error)
+            #
+            continue
+        #
+        try:
+            yield ExclusionRule(current_line, style=current_style)
+        except ValueError as value_error:
+            warning(str(value_error))
+        #
+    #
+
+
 def translate(pattern: str, style: Style = Style.GITIGNORE) -> str:
     """Dispatch pattern to the appropriate translation function,
     according to the style argument
@@ -218,7 +334,7 @@ def translate(pattern: str, style: Style = Style.GITIGNORE) -> str:
     match style:
         case Style.GITIGNORE:
             return gitignore_pattern_to_full_regex(pattern)
-        case Style.REGEX:
+        case Style.REGEXP:
             return pattern
         case _:
             raise NotImplementedError
@@ -366,7 +482,7 @@ def dissect_glob_segment(
         the sequence of regex chunks and the sequence of star positions
             in the chunks sequence.
             This result is post-processed in the
-            [glob_segment_to_regex_segment][offgrep.ignore.glob_segment_to_regex_segment]
+            [glob_segment_to_regex_segment][offgrep.path_matching.glob_segment_to_regex_segment]
             function which calls `dissect_glob_segment()` internally.
     """
     re_chunks: list[str] = []
@@ -431,7 +547,7 @@ def bracketed_glob_pattern_part_to_partial_regex(
 
     !!! note
         Factored out from the
-        [dissect_glob_segment()][offgrep.ignore.dissect_glob_segment]
+        [dissect_glob_segment()][offgrep.path_matching.dissect_glob_segment]
         function
 
     Args:

offgrep-0.1.0/src/offgrep/filesystem.py

@@ -1,182 +0,0 @@
-# -*- coding: utf-8 -*-
-
-"""
-Filesystem related functionality
-"""
-
-from collections.abc import Iterator
-from logging import debug
-from pathlib import Path
-
-from .git import GIT_DIR, get_configured_global_gitignore_path
-from .ignore import CompiledPattern, IgnoreFile
-
-
-GIT_IGNORE_FILE = ".gitignore"
-
-DEFAULT_IGNORE_FILES = (GIT_IGNORE_FILE,)
-
-
-class IgnoredDirectoryError(Exception):
-    """raised if a directory was explicitly excluded"""
-
-
-def load_gitignore_file(path: Path, handles: Path, description: str) -> IgnoreFile:
-    """load an ignore file"""
-    if path.is_file():
-        return IgnoreFile.from_gitignore(path, handles)
-    #
-    raise ValueError(f"{description} ignore file {path} not found")
-
-
-def iter_target_files(start_directory: Path) -> Iterator[Path]:
-    """Iterate over target files"""
-    absolute_start = start_directory.resolve()
-    debug("Compiling ignore files for directory %s …", absolute_start)
-    ignore_files = compile_ignore_files(absolute_start)
-    debug("Walking down directory %s", absolute_start)
-    yield from walk_files(absolute_start, absolute_start, ignore_files)
-
-
-# pylint: disable=too-many-locals ; refactoring candidate
-
-
-def compile_ignore_files(
-    start_directory: Path, check_git: bool = True
-) -> list[IgnoreFile]:
-    """Compile ignore files for start_directory"""
-    absolute_start = start_directory.resolve()
-    assert absolute_start.is_dir()
-    git_ignore_files: list[IgnoreFile] = []
-    if check_git:
-        dirs_to_check = [absolute_start, *absolute_start.parents]
-        checked_dirs = []
-        for pointer in dirs_to_check:
-            debug("Checking %s for %s …", pointer, GIT_DIR)
-            checked_dirs.append(pointer)
-            candidate = pointer / GIT_DIR
-            if candidate.is_dir():
-                debug("… found! Compiling ignore rules …")
-                git_ignore_files.append(
-                    IgnoreFile(
-                        handles_directory=pointer,
-                        rules=(CompiledPattern.from_gitignore_line(f"{GIT_DIR}/"),),
-                    )
-                )
-                for description, file_path in (
-                    ("user global", get_configured_global_gitignore_path()),
-                    ("repo private", candidate / "info" / "exclude"),
-                ):
-                    try:
-                        git_ignore_files.append(
-                            load_gitignore_file(file_path, pointer, description)
-                        )
-                    except (OSError, ValueError) as error:
-                        debug(str(error))
-                    #
-                #
-                break
-            #
-        else:
-            debug("No git repository found in any parent, continuing without …")
-        #
-        # walk back up and handle intermediate ignore files,
-        # except for the start directory itself (that is done when walking it)
-        if git_ignore_files:
-            for idx in range(len(checked_dirs) - 1, 0, -1):
-                current_dir = checked_dirs[idx]
-                try:
-                    intermediate_ignore_file = load_gitignore_file(
-                        current_dir / GIT_IGNORE_FILE,
-                        current_dir,
-                        "intermediate",
-                    )
-                except (OSError, ValueError) as error:
-                    debug(str(error))
-                else:
-                    git_ignore_files.append(intermediate_ignore_file)
-                #
-                dir_to_check = checked_dirs[idx - 1]
-                is_excluded = False
-                for current_ignore_file in git_ignore_files:
-                    is_excluded = current_ignore_file.excludes_path(
-                        dir_to_check, pre_excluded=is_excluded
-                    )
-                #
-                if is_excluded:
-                    raise IgnoredDirectoryError(
-                        f"gitignore config excludes directory {dir_to_check},"
-                        " cannot continue"
-                    )
-                #
-            #
-        #
-    #
-    return git_ignore_files
-
-
-def walk_files(
-    start_directory: Path,
-    working_directory: Path,
-    ignore_files: list[IgnoreFile],
-    use_ignore_files: tuple[str, ...] = DEFAULT_IGNORE_FILES,
-) -> Iterator[Path]:
-    """Walk through all contained files, respecting the ignore rules.
-
-    Yield paths relative to the start directory if the start directory
-    is the current working directory.
-    """
-    assert start_directory.is_absolute()
-    use_relative_paths = False
-    cwd = Path.cwd()
-    anchor_directory = cwd
-    if start_directory.is_relative_to(Path.cwd()):
-        use_relative_paths = True
-    #
-    absolute_workdir = working_directory.resolve()
-    effective_ignore_files = list(ignore_files)
-    for ignore_file_candidate in use_ignore_files:
-        try:
-            effective_ignore_files.append(
-                load_gitignore_file(
-                    absolute_workdir / ignore_file_candidate,
-                    absolute_workdir,
-                    "Directory level",
-                )
-            )
-        except (OSError, ValueError) as error:
-            debug(str(error))
-        #
-    #
-    for contained_path in sorted(absolute_workdir.glob("*"), key=Path.as_posix):
-        # name = contained_path.name
-        if not (contained_path.is_dir() or contained_path.is_file()):
-            debug("Skipping %s, not a directory or regular file", contained_path)
-            continue
-        #
-        is_excluded = False
-        for current_ignore_file in effective_ignore_files:
-            is_excluded = current_ignore_file.excludes_path(
-                contained_path, pre_excluded=is_excluded
-            )
-        #
-        if is_excluded:
-            continue
-        #
-        if contained_path.is_file():
-            if use_relative_paths:
-                yield contained_path.relative_to(anchor_directory)
-            else:
-                yield contained_path
-            #
-        #
-        if contained_path.is_dir():
-            debug("Descending into directory %s", contained_path.name)
-            yield from walk_files(
-                start_directory,
-                contained_path,
-                effective_ignore_files,
-                use_ignore_files=use_ignore_files,
-            )
-        #
-    #