todex 1.0.0__tar.gz

todex-1.0.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Kaloyan Ivanov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

todex-1.0.0/PKG-INFO

@@ -0,0 +1,23 @@
+Metadata-Version: 2.4
+Name: todex
+Version: 1.0.0
+Summary: TODO extractor
+Keywords: TODO extractor,cli tool
+Author-Email: kaliv0 <kaliv.zero@gmail.com>
+License-File: LICENSE
+Project-URL: repository, https://github.com/kaliv0/todex.git
+Requires-Python: >=3.12
+Requires-Dist: pathspec>=1.1.1
+Description-Content-Type: text/markdown
+
+<div align="center">
+  <img src="https://github.com/kaliv0/todex/blob/main/assets/truck.jpg?raw=true" width="400" alt="Todex">
+</div>
+
+# todex
+
+![Python 3.X](https://img.shields.io/badge/python-^3.12-blue?style=flat-square&logo=Python&logoColor=white)
+[![PyPI](https://img.shields.io/pypi/v/todex.svg)](https://pypi.org/project/todex/)
+[![Downloads](https://static.pepy.tech/badge/todex)](https://pepy.tech/projects/todex)
+
+Peak mud digger and TODO extractor

todex-1.0.0/README.md

@@ -0,0 +1,11 @@
+<div align="center">
+  <img src="https://github.com/kaliv0/todex/blob/main/assets/truck.jpg?raw=true" width="400" alt="Todex">
+</div>
+
+# todex
+
+![Python 3.X](https://img.shields.io/badge/python-^3.12-blue?style=flat-square&logo=Python&logoColor=white)
+[![PyPI](https://img.shields.io/pypi/v/todex.svg)](https://pypi.org/project/todex/)
+[![Downloads](https://static.pepy.tech/badge/todex)](https://pepy.tech/projects/todex)
+
+Peak mud digger and TODO extractor

todex-1.0.0/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "todex"
+version = "1.0.0"
+description = "TODO extractor"
+keywords = [
+    "TODO extractor",
+    "cli tool",
+]
+authors = [
+    { name = "kaliv0", email = "kaliv.zero@gmail.com" },
+]
+license-files = [
+    "LICENSE",
+]
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "pathspec>=1.1.1",
+]
+
+[project.urls]
+repository = "https://github.com/kaliv0/todex.git"
+
+[project.scripts]
+todex = "todex.__main__:main"
+
+[dependency-groups]
+dev = [
+    "mypy>=2.3.1",
+    "ruff>=0.16.8",
+]
+
+[build-system]
+build-backend = "pdm.backend"
+requires = [
+    "pdm-backend",
+]
+
+[tool.ruff]
+fix = true
+line-length = 100

todex-1.0.0/todex/__main__.py

@@ -0,0 +1,145 @@
+import sys
+from argparse import ArgumentParser, Namespace, RawTextHelpFormatter
+
+from todex.extractor import DEFAULT_MAX_DEPTH, DEFAULT_OUT, Extractor
+
+__version__ = "1.0.0"
+
+TRASH = r"""
+       ________________   ___/-\___     ___/-\___     ___/-\___
+     / /             ||  |---------|   |---------|   |---------|
+    / /              ||   |       |     | | | | |     |   |   |
+   / /             __||   |       |     | | | | |     | | | | |
+  / /   \\        I  ||   |       |     | | | | |     | | | | |
+ (-------------------||   | | | | |     | | | | |     | | | | |
+ ||               == ||   |_______|     |_______|     |_______|
+ ||       TODEX      | =============================================
+ ||          ____    |                                ____      |
+( | o      / ____ \                                 / ____ \    |)
+ ||      / / . . \ \                              / / . . \ \   |
+[ |_____| | .   . | |____________________________| | .   . | |__]
+          | .   . |                                | .   . |
+           \_____/                                  \_____/
+
+            Peak trash sniffer and TODO extractor
+"""
+
+
+class NoUsageFormatter(RawTextHelpFormatter):
+    def _format_usage(self, usage, actions, groups, prefix):
+        return ""
+
+
+class ArgValidator:
+    def __init__(self, parser: ArgumentParser, args: Namespace) -> None:
+        self.parser = parser
+        self.args = args
+
+    def require(self, flag_attr: str, needs_attr: str, flag: str, needs: str) -> None:
+        if getattr(self.args, flag_attr) and not getattr(self.args, needs_attr):
+            self.parser.error(f"{flag} requires {needs}")
+
+
+def main() -> None:
+    parser = ArgumentParser(
+        prog="todex",
+        formatter_class=NoUsageFormatter,
+        description=TRASH,
+    )
+    parser.add_argument("path", metavar="PATH", help="path to file or dir to process")
+    parser.add_argument(
+        "-x",
+        "--exclude",
+        nargs="*",
+        default=[],
+        metavar="PATTERN",
+        help="""git-like patterns to skip (paths, names, or globs together with default .* / __*), e.g.
+
+    bar.py       any file named bar.py
+    /bar.py      only top-level
+    foo/bar.py   path under the scan root
+    vendor/      directory and its contents
+    **/*.pyc     nested matches
+    !fizz.md     re-include after a broader exclude
+
+""",
+    )
+    parser.add_argument(
+        "-H",
+        "--include-hidden",
+        action="store_true",
+        help="do not apply default excludes for hidden names (.* / __*)",
+    )
+    parser.add_argument(
+        "-t",
+        "--tokens",
+        nargs="*",
+        default=[],
+        metavar="TOKEN",
+        help="""list of tokens to search for (together with default TODO, FIXME) e.g. WARN, REVISIT.
+If the token is followed by {lines-count} e.g. #FIXME{3}
+the extractor will include multiline snippet with the length specified between the curly braces:
+
+    #FIXME{3} - revist after release
+    if self.foo == "bar":
+        return f"fizz{buzz}"
+
+""",
+    )
+    parser.add_argument(
+        "-i",
+        "--ignore-default",
+        action="store_true",
+        help="use only -t tokens (skip default TODO, FIXME)",
+    )
+    parser.add_argument(
+        "-f", "--full-path", action="store_true", help="display absolute dir/file path"
+    )
+    parser.add_argument(
+        "-s",
+        "--short",
+        action="store_true",
+        help="display only token messages e.g. '#TODO:  after release', ignore longer snippets",
+    )
+    parser.add_argument(
+        "-r",
+        "--recursive",
+        action="store_true",
+        help="traverse given dir path recursively",
+    )
+    parser.add_argument(
+        "-m",
+        "--max-depth",
+        type=int,
+        metavar="N",
+        default=DEFAULT_MAX_DEPTH,
+        help="maximum depth of dir traversal - used with -r/--recursive flag",
+    )
+    parser.add_argument(
+        "-d",
+        "--debug",
+        action="store_true",
+        help="enable debug mode",
+    )
+    parser.add_argument(
+        "-o",
+        "--out",
+        default=DEFAULT_OUT,
+        help="path to output file, if existing dir is passed instead - TODO out file will be saved inside",
+    )
+    parser.add_argument("-v", "--version", action="version", version=f"%(prog)s {__version__}")
+    args = parser.parse_args()
+
+    validator = ArgValidator(parser, args)
+    validator.require("ignore_default", "tokens", "-i/--ignore-default", "-t/--tokens")
+    try:
+        Extractor(**vars(args)).run()
+    except KeyboardInterrupt:
+        sys.exit(130)
+    except (OSError, TypeError, ValueError) as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

todex-1.0.0/todex/extractor.py

@@ -0,0 +1,181 @@
+import re
+import sys
+from functools import cached_property
+from pathlib import Path
+from re import Pattern
+from textwrap import dedent, indent
+
+from pathspec import PathSpec
+
+from todex.writer import Writer
+
+DEFAULT_MAX_DEPTH = float("inf")
+DEFAULT_OUT = "TODO"
+DEFAULT_GLOBS = [".*", "__*"]
+DEFAULT_TOKENS = ["TODO", "FIXME"]
+
+
+class Extractor:
+    def __init__(
+        self,
+        path,
+        exclude: list[str] | None = None,
+        include_hidden: bool = False,
+        tokens: list[str] | None = None,
+        ignore_default: bool = False,
+        out: str = DEFAULT_OUT,
+        full_path: bool = False,
+        short: bool = False,
+        recursive: bool = False,
+        max_depth: float = DEFAULT_MAX_DEPTH,
+        debug: bool = False,
+    ) -> None:
+        self.root = Path(path)
+        self.exclude_spec = self.prepare_exclude_spec(exclude or [], include_hidden)
+        self.token_pattern = self.prepare_token_pattern(tokens or [], ignore_default)
+        self.out = self.prepare_out(out)
+        self.full_path = full_path
+        self.short = short
+        self.recursive = recursive
+        self.max_depth = max_depth
+        self.debug = __debug__ and debug
+
+    @staticmethod
+    def prepare_exclude_spec(exclude: list[str], include_hidden: bool) -> PathSpec:
+        if not include_hidden:
+            exclude = [*DEFAULT_GLOBS, *exclude]
+        exclude = list(dict.fromkeys(exclude))  # dedupe + keep order
+        return PathSpec.from_lines("gitignore", exclude)
+
+    @staticmethod
+    def prepare_token_pattern(tokens: list[str], ignore_default: bool) -> Pattern:
+        if not ignore_default:
+            tokens = [*DEFAULT_TOKENS, *tokens]
+        if not tokens:
+            # prevented by CLI (-i without -t) but you can never be too carefull
+            raise ValueError("no tokens to search for")
+        tokens = sorted(dict.fromkeys(tokens), key=len, reverse=True)
+        alternations = "|".join(re.escape(tok) for tok in tokens)
+        return re.compile(rf"(?<!\w)(?:{alternations})(?!\w)", re.IGNORECASE)
+
+    @staticmethod
+    def prepare_out(name: str) -> Writer:
+        if (path := Path(name)).is_dir():
+            path = path / DEFAULT_OUT
+        return Writer(path)
+
+    @cached_property
+    def out_path(self) -> Path:
+        return self.out.path.absolute()
+
+    def run(self) -> None:
+        if not self.root.exists():
+            raise FileNotFoundError(f"{self.root} not found")
+
+        if self.debug:
+            print("processing:")
+        try:
+            if self.root.is_file():
+                self.process_file(self.root)
+            elif self.root.is_dir():
+                self.process_dir(self.root, depth=0)
+            else:
+                raise ValueError(f"{self.root} must be path to file or dir")
+        finally:
+            self.out.close()
+
+    def process_file(self, path: Path) -> None:
+        if self.is_excluded(path):
+            return
+
+        if self.debug:
+            print(f"\t{self.prepare_path_name(path)}")
+
+        try:
+            with path.open() as f:
+                lines_count = 0
+                header_added = False
+                while line := f.readline():
+                    lines_count += 1
+
+                    if not (match := self.token_pattern.search(line)):
+                        continue
+                    token = match.group()
+                    idx = match.start()
+
+                    if not header_added:
+                        self.out.write_header(self.prepare_path_name(path))
+                        header_added = True
+
+                    if self.short:
+                        line = line[idx:]
+                        self.out.write_entry(lines_count, line)
+                        continue
+
+                    display_line_num = lines_count
+                    for _ in range(1, self.get_snippet_count(line, start=idx + len(token))):
+                        if not (next_line := f.readline()):
+                            break
+                        line += next_line
+                        lines_count += 1
+
+                    if not line.endswith("\n"):
+                        line += "\n"  # this is the last or only line in the file
+
+                    if display_line_num != lines_count:  # add full snippets
+                        line = indent(dedent("\n" + line), "\t\t")
+                    else:
+                        line = dedent(line)
+                    self.out.write_entry(display_line_num, line)
+        except UnicodeDecodeError as e:
+            print(f"skipping {self.prepare_path_name(path)}:\n{e}", file=sys.stderr)
+
+    @staticmethod
+    def get_snippet_count(line: str, start: int) -> int:
+        if not line.startswith("{", start):
+            return 1
+
+        # advance pointer to actual count
+        start = start + 1
+        if (end := line[start:].find("}")) == -1:
+            raise ValueError("invalid snippet count")
+        # cut snippet line count and try parse to int -> if it fails global error handler will log the error
+        return int(line[start : start + end])
+
+    def prepare_path_name(self, path: Path) -> str:
+        if self.full_path:
+            return str(path.absolute())
+        if self.recursive:
+            return str(path)
+        return path.name
+
+    def process_dir(self, path: Path, depth: int) -> None:
+        if self.is_excluded(path, is_dir=True):
+            return
+
+        if depth > self.max_depth:
+            return
+
+        dirs = []
+        for entry in path.iterdir():
+            if entry.is_file():
+                self.process_file(entry)
+            elif self.recursive and entry.is_dir():
+                dirs.append(entry)
+
+        for dir_ in dirs:
+            self.process_dir(dir_, depth + 1)
+
+    def is_excluded(self, path: Path, is_dir: bool = False) -> bool:
+        # always process scan root
+        if path == self.root:
+            return False
+
+        # skip summary file
+        if path.absolute() == self.out_path:
+            return True
+
+        rel_path = path.relative_to(self.root).as_posix()
+        if is_dir:
+            return any(self.exclude_spec.match_files([rel_path, rel_path + "/"]))
+        return self.exclude_spec.match_file(rel_path)

todex-1.0.0/todex/writer.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+from typing import TextIO
+
+
+class Writer:
+    def __init__(self, path: Path) -> None:
+        self.path = path
+        self.file: TextIO | None = None
+
+    def ensure_open(self) -> TextIO:
+        if self.file is None:
+            self.file = self.path.open("w")
+        return self.file
+
+    def close(self) -> None:
+        if self.file:
+            self.file.close()
+            self.file = None
+
+    def write_header(self, path: str) -> None:
+        f = self.ensure_open()
+        f.write("======================\n")
+        f.write(f"{path}\n")
+
+    def write_entry(self, lines_count: int, line: str) -> None:
+        f = self.ensure_open()
+        f.write(f"\tline {lines_count}: ")
+        f.write(line)