repo-parser 0.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- repo_parser/__init__.py +16 -0
- repo_parser/cli.py +163 -0
- repo_parser/filesystem.py +120 -0
- repo_parser/processor.py +14 -0
- repo_parser/resource.py +285 -0
- repo_parser/templates/idr.md +91 -0
- repo_parser-0.0.3.dist-info/METADATA +92 -0
- repo_parser-0.0.3.dist-info/RECORD +11 -0
- repo_parser-0.0.3.dist-info/WHEEL +4 -0
- repo_parser-0.0.3.dist-info/entry_points.txt +3 -0
- repo_parser-0.0.3.dist-info/licenses/LICENSE +21 -0
repo_parser/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""
|
|
2
|
+
A library for extracting metadata out of a source repository.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
__version__ = "0.0.3"
|
|
6
|
+
|
|
7
|
+
from .filesystem import scan
|
|
8
|
+
from .processor import Processor
|
|
9
|
+
from .resource import Resource, get_resources
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"Processor",
|
|
13
|
+
"Resource",
|
|
14
|
+
"get_resources",
|
|
15
|
+
"scan",
|
|
16
|
+
]
|
repo_parser/cli.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Command-line interface for repo-parser.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import pathlib
|
|
7
|
+
import re
|
|
8
|
+
from datetime import UTC, datetime
|
|
9
|
+
from importlib import resources
|
|
10
|
+
|
|
11
|
+
import git
|
|
12
|
+
import jinja2
|
|
13
|
+
import typer
|
|
14
|
+
from slugify import slugify
|
|
15
|
+
|
|
16
|
+
app = typer.Typer(help="repo-parser CLI tools")
|
|
17
|
+
idr_app = typer.Typer(help="IDR (Implementation Decision Record) commands")
|
|
18
|
+
app.add_typer(idr_app, name="idr")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _get_git_author() -> str:
|
|
22
|
+
"""
|
|
23
|
+
Get the author name and email from git config.
|
|
24
|
+
|
|
25
|
+
Returns:
|
|
26
|
+
Author in format "Name <email>", or "Unknown" if not set.
|
|
27
|
+
"""
|
|
28
|
+
try:
|
|
29
|
+
# Read from global git config (doesn't require being in a repo)
|
|
30
|
+
config = git.GitConfigParser(
|
|
31
|
+
[git.config.get_config_path("global")], read_only=True
|
|
32
|
+
)
|
|
33
|
+
name = config.get_value("user", "name")
|
|
34
|
+
email = config.get_value("user", "email")
|
|
35
|
+
except (KeyError, ValueError, OSError, git.GitCommandNotFound):
|
|
36
|
+
# Config file doesn't exist, user.name/email not set, or git not installed
|
|
37
|
+
typer.echo(
|
|
38
|
+
"Warning: Could not read git config user.name/email, using 'Unknown' as author",
|
|
39
|
+
err=True,
|
|
40
|
+
)
|
|
41
|
+
return "Unknown"
|
|
42
|
+
else:
|
|
43
|
+
return f"{name} <{email}>"
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _get_repo_root() -> pathlib.Path:
|
|
47
|
+
"""
|
|
48
|
+
Get the root directory of the git repository.
|
|
49
|
+
|
|
50
|
+
Returns:
|
|
51
|
+
Path to repository root.
|
|
52
|
+
|
|
53
|
+
Raises:
|
|
54
|
+
typer.Exit: If not in a git repository.
|
|
55
|
+
"""
|
|
56
|
+
try:
|
|
57
|
+
repo = git.Repo(search_parent_directories=True)
|
|
58
|
+
return pathlib.Path(repo.working_dir)
|
|
59
|
+
except git.InvalidGitRepositoryError as e:
|
|
60
|
+
typer.echo("Error: Not in a git repository", err=True)
|
|
61
|
+
raise typer.Exit(1) from e
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _load_template() -> str:
|
|
65
|
+
"""
|
|
66
|
+
Load the IDR template from package resources.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
Template content as string.
|
|
70
|
+
"""
|
|
71
|
+
# Python 3.11+ uses importlib.resources.files
|
|
72
|
+
template_path = resources.files("repo_parser").joinpath("templates/idr.md")
|
|
73
|
+
return template_path.read_text()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _strip_html_comments(text: str) -> str:
|
|
77
|
+
"""
|
|
78
|
+
Strip HTML comments from text.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
text: Text containing HTML comments.
|
|
82
|
+
|
|
83
|
+
Returns:
|
|
84
|
+
Text with HTML comments removed and extra blank lines cleaned up.
|
|
85
|
+
"""
|
|
86
|
+
# Remove HTML comments (including multiline)
|
|
87
|
+
# Note: This doesn't handle edge cases like --> within comment text,
|
|
88
|
+
# but that's unlikely in our IDR templates
|
|
89
|
+
text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
|
|
90
|
+
# Clean up any blank lines left by removing comments
|
|
91
|
+
# First normalize all whitespace-only lines to actual blank lines
|
|
92
|
+
text = re.sub(r"^\s+$", "", text, flags=re.MULTILINE)
|
|
93
|
+
# Then collapse multiple consecutive blank lines (3+) to exactly 2 newlines
|
|
94
|
+
text = re.sub(r"\n{3,}", "\n\n", text)
|
|
95
|
+
# Finally, strip any leading/trailing whitespace from the entire document
|
|
96
|
+
return text.strip()
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@idr_app.command("new")
|
|
100
|
+
def idr_new(
|
|
101
|
+
title: str,
|
|
102
|
+
no_comments: bool = typer.Option(
|
|
103
|
+
False,
|
|
104
|
+
"--no-comments",
|
|
105
|
+
help="Create IDR without explanatory comments in the template",
|
|
106
|
+
),
|
|
107
|
+
):
|
|
108
|
+
"""
|
|
109
|
+
Create a new IDR (Implementation Decision Record).
|
|
110
|
+
|
|
111
|
+
Creates a new IDR file in the idrs/ directory with a timestamp and slugified title.
|
|
112
|
+
Example: rp idr new "Add last modified metadata"
|
|
113
|
+
"""
|
|
114
|
+
repo_root = _get_repo_root()
|
|
115
|
+
|
|
116
|
+
# Create idrs directory if it doesn't exist
|
|
117
|
+
idrs_dir = repo_root / "idrs"
|
|
118
|
+
idrs_dir.mkdir(exist_ok=True)
|
|
119
|
+
|
|
120
|
+
# Generate timestamp in UTC
|
|
121
|
+
timestamp = datetime.now(UTC).strftime("%Y%m%d%H%M")
|
|
122
|
+
|
|
123
|
+
# Slugify the title
|
|
124
|
+
slug = slugify(title)
|
|
125
|
+
|
|
126
|
+
# Create filename
|
|
127
|
+
filename = f"{timestamp}-{slug}.md"
|
|
128
|
+
filepath = idrs_dir / filename
|
|
129
|
+
|
|
130
|
+
# Check if file already exists (realistically should not happen)
|
|
131
|
+
if filepath.exists():
|
|
132
|
+
typer.echo(f"Error: File already exists: {filepath}", err=True)
|
|
133
|
+
raise typer.Exit(1)
|
|
134
|
+
|
|
135
|
+
# Get author from git config
|
|
136
|
+
author = _get_git_author()
|
|
137
|
+
|
|
138
|
+
# Load and render template
|
|
139
|
+
template_content = _load_template()
|
|
140
|
+
|
|
141
|
+
# Check if we should strip comments (from flag or environment variable)
|
|
142
|
+
strip_comments = no_comments or os.environ.get(
|
|
143
|
+
"RP_IDR_NO_COMMENTS", ""
|
|
144
|
+
).lower() in (
|
|
145
|
+
"1",
|
|
146
|
+
"true",
|
|
147
|
+
"yes",
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
if strip_comments:
|
|
151
|
+
template_content = _strip_html_comments(template_content)
|
|
152
|
+
|
|
153
|
+
template = jinja2.Template(template_content)
|
|
154
|
+
rendered = template.render(title=title, author=author)
|
|
155
|
+
|
|
156
|
+
# Write file
|
|
157
|
+
filepath.write_text(rendered)
|
|
158
|
+
|
|
159
|
+
typer.echo(f"Created IDR: {filepath}")
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
if __name__ == "__main__":
|
|
163
|
+
app()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
import pathlib
|
|
2
|
+
import re
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from pathlib import PurePath
|
|
5
|
+
|
|
6
|
+
import git
|
|
7
|
+
|
|
8
|
+
from .processor import Processor
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dataclass
|
|
12
|
+
class File:
|
|
13
|
+
name: str
|
|
14
|
+
src_path: PurePath
|
|
15
|
+
content: str | None
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@dataclass
|
|
19
|
+
class Dir:
|
|
20
|
+
path: pathlib.Path
|
|
21
|
+
files: list[File]
|
|
22
|
+
dirs: list["Dir"]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _scan(
|
|
26
|
+
path: pathlib.Path,
|
|
27
|
+
processors: list[Processor],
|
|
28
|
+
ignore_patterns: list[re.Pattern],
|
|
29
|
+
repo: git.Repo,
|
|
30
|
+
depth: int,
|
|
31
|
+
max_depth: int | None,
|
|
32
|
+
) -> Dir:
|
|
33
|
+
dir = Dir(path=path, files=[], dirs=[])
|
|
34
|
+
entries = list(path.iterdir())
|
|
35
|
+
if not entries:
|
|
36
|
+
return dir
|
|
37
|
+
resolved_entries = {str(entry.resolve()) for entry in entries}
|
|
38
|
+
|
|
39
|
+
# remove any entries which match our ignore patterns (e.g. autogenerated files)
|
|
40
|
+
ignored = {
|
|
41
|
+
resolved_entry
|
|
42
|
+
for resolved_entry in resolved_entries
|
|
43
|
+
if any(
|
|
44
|
+
re.search(pattern, resolved_entry) is not None
|
|
45
|
+
for pattern in ignore_patterns
|
|
46
|
+
)
|
|
47
|
+
}
|
|
48
|
+
# also ignore anything .gitignored (doing this second because it's a likely a little
|
|
49
|
+
# more time consuming)
|
|
50
|
+
remaining_entries = resolved_entries - ignored
|
|
51
|
+
if remaining_entries:
|
|
52
|
+
ignored |= set(repo.ignored(*remaining_entries))
|
|
53
|
+
|
|
54
|
+
for entry in entries:
|
|
55
|
+
# FIXME: At some point we should only ignore .git in the root directory
|
|
56
|
+
if entry.name == ".git" or str(entry.resolve()) in ignored:
|
|
57
|
+
continue
|
|
58
|
+
if entry.is_file():
|
|
59
|
+
for processor in processors:
|
|
60
|
+
if processor.pattern.search(str(entry)):
|
|
61
|
+
content = entry.read_text() if processor.read_content else None
|
|
62
|
+
dir.files.append(
|
|
63
|
+
File(
|
|
64
|
+
name=entry.name,
|
|
65
|
+
src_path=(path / entry.name),
|
|
66
|
+
content=content,
|
|
67
|
+
)
|
|
68
|
+
)
|
|
69
|
+
elif entry.is_dir() and (max_depth is None or depth < max_depth):
|
|
70
|
+
dir.dirs.append(
|
|
71
|
+
_scan(
|
|
72
|
+
path / entry.name,
|
|
73
|
+
processors,
|
|
74
|
+
ignore_patterns,
|
|
75
|
+
repo,
|
|
76
|
+
depth + 1,
|
|
77
|
+
max_depth,
|
|
78
|
+
)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
return dir
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def scan(
|
|
85
|
+
path: pathlib.Path,
|
|
86
|
+
processors: list[Processor],
|
|
87
|
+
ignore_patterns: list[re.Pattern] | None = None,
|
|
88
|
+
subdirs: list[pathlib.Path] | None = None,
|
|
89
|
+
max_depth: int | None = None,
|
|
90
|
+
) -> tuple[Dir, git.Repo]:
|
|
91
|
+
"""
|
|
92
|
+
Scans a GitHub repository for files and subdirectories, returning a data
|
|
93
|
+
structure representing the directory tree.
|
|
94
|
+
|
|
95
|
+
Takes in a list of processors to figure out which files should be included
|
|
96
|
+
in the returned tree.
|
|
97
|
+
|
|
98
|
+
Under the hood, this uses the `git` library to scan the repository, skipping
|
|
99
|
+
files in .gitignore.
|
|
100
|
+
|
|
101
|
+
This step does not do any post-processing of the files, though their
|
|
102
|
+
content is read in if one of their processors requires it.
|
|
103
|
+
|
|
104
|
+
Returns a tuple of (Dir, git.Repo) for further processing.
|
|
105
|
+
"""
|
|
106
|
+
repo = git.Repo(path, search_parent_directories=True)
|
|
107
|
+
|
|
108
|
+
# if subdirs, just scan each one and return the results
|
|
109
|
+
if subdirs:
|
|
110
|
+
dir = Dir(path=path, files=[], dirs=[])
|
|
111
|
+
for subdir in subdirs:
|
|
112
|
+
dir.dirs.append(
|
|
113
|
+
_scan(
|
|
114
|
+
path / subdir, processors, ignore_patterns or [], repo, 0, max_depth
|
|
115
|
+
)
|
|
116
|
+
)
|
|
117
|
+
return dir, repo
|
|
118
|
+
|
|
119
|
+
# otherwise read everything up to max_depth
|
|
120
|
+
return _scan(path, processors, ignore_patterns or [], repo, 0, max_depth), repo
|
repo_parser/processor.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import re
|
|
2
|
+
from collections.abc import Callable
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class Processor:
|
|
8
|
+
"""
|
|
9
|
+
Processor class to process files based on a pattern.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
pattern: re.Pattern
|
|
13
|
+
process: Callable
|
|
14
|
+
read_content: bool
|
repo_parser/resource.py
ADDED
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
from datetime import datetime
|
|
3
|
+
from pathlib import Path, PurePath
|
|
4
|
+
|
|
5
|
+
import git
|
|
6
|
+
|
|
7
|
+
from .filesystem import Dir
|
|
8
|
+
from .processor import Processor
|
|
9
|
+
|
|
10
|
+
# Chunk size for batch git log queries to avoid command-line length limits
|
|
11
|
+
LAST_MODIFIED_CHUNK_SIZE = 200
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class Resource:
|
|
16
|
+
name: str
|
|
17
|
+
src_path: PurePath
|
|
18
|
+
path: PurePath
|
|
19
|
+
type: str
|
|
20
|
+
metadata: dict
|
|
21
|
+
content: str | None
|
|
22
|
+
children: list["Resource"]
|
|
23
|
+
last_modified: datetime
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _get_last_modified(repo: git.Repo, file_path: PurePath) -> datetime:
|
|
27
|
+
"""Get the last modification date from git as a datetime object"""
|
|
28
|
+
commits = list(repo.iter_commits(paths=str(file_path), max_count=1))
|
|
29
|
+
if commits:
|
|
30
|
+
return datetime.fromtimestamp(commits[0].committed_date)
|
|
31
|
+
|
|
32
|
+
return datetime.now()
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _get_last_modified_batch(
|
|
36
|
+
repo: git.Repo, file_paths: list[PurePath], scan_root: Path | None = None
|
|
37
|
+
) -> dict[PurePath, datetime]:
|
|
38
|
+
"""
|
|
39
|
+
Get the last modification dates for multiple files in a single batched git call.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
repo: The git repository
|
|
43
|
+
file_paths: List of file paths to query (can be absolute or relative)
|
|
44
|
+
scan_root: Optional root path for resolving relative file paths. If provided,
|
|
45
|
+
relative paths will be resolved against this root. If None, relative paths
|
|
46
|
+
are resolved against the current working directory or repo root.
|
|
47
|
+
|
|
48
|
+
Returns:
|
|
49
|
+
Dictionary mapping file paths to their last commit dates. Files with no
|
|
50
|
+
git history will have datetime.now() as their value.
|
|
51
|
+
|
|
52
|
+
Raises:
|
|
53
|
+
ValueError: If a file path cannot be converted to a relative path
|
|
54
|
+
git.GitCommandError: If git log command fails (may be raised by repo.git.log)
|
|
55
|
+
"""
|
|
56
|
+
if not file_paths:
|
|
57
|
+
return {}
|
|
58
|
+
|
|
59
|
+
repo_root = Path(repo.working_dir)
|
|
60
|
+
result: dict[PurePath, datetime] = {}
|
|
61
|
+
|
|
62
|
+
# Convert all paths to relative paths and build a mapping
|
|
63
|
+
# We need to map relative paths back to original PurePath objects
|
|
64
|
+
rel_path_to_original: dict[str, PurePath] = {}
|
|
65
|
+
rel_paths: list[str] = []
|
|
66
|
+
|
|
67
|
+
for file_path in file_paths:
|
|
68
|
+
# Convert to Path and resolve to absolute path
|
|
69
|
+
path_obj = Path(file_path)
|
|
70
|
+
if path_obj.is_absolute():
|
|
71
|
+
abs_path = path_obj
|
|
72
|
+
else:
|
|
73
|
+
# Try resolving relative path against scan_root (if provided) or current working directory
|
|
74
|
+
if scan_root is not None:
|
|
75
|
+
abs_path = (scan_root / path_obj).resolve()
|
|
76
|
+
else:
|
|
77
|
+
abs_path = path_obj.resolve()
|
|
78
|
+
|
|
79
|
+
# If the resolved path is not within the repo root, try resolving against repo root
|
|
80
|
+
try:
|
|
81
|
+
abs_path.relative_to(repo_root)
|
|
82
|
+
except ValueError:
|
|
83
|
+
# Path is not within repo when resolved against scan_root/cwd, try repo root
|
|
84
|
+
abs_path = (repo_root / path_obj).resolve()
|
|
85
|
+
|
|
86
|
+
# Note: In normal operation, all paths come from scanning the repository,
|
|
87
|
+
# so they should always be within the repo root. This may raise ValueError
|
|
88
|
+
# if a path is outside the repo, but that shouldn't happen in practice.
|
|
89
|
+
rel_path = str(abs_path.relative_to(repo_root))
|
|
90
|
+
|
|
91
|
+
# Normalize path separators (git uses forward slashes)
|
|
92
|
+
rel_path = rel_path.replace("\\", "/")
|
|
93
|
+
rel_path_to_original[rel_path] = file_path
|
|
94
|
+
rel_paths.append(rel_path)
|
|
95
|
+
|
|
96
|
+
# Process in chunks to avoid command-line length limits
|
|
97
|
+
for i in range(0, len(rel_paths), LAST_MODIFIED_CHUNK_SIZE):
|
|
98
|
+
chunk = rel_paths[i : i + LAST_MODIFIED_CHUNK_SIZE]
|
|
99
|
+
chunk_set = set(chunk) # For fast lookup
|
|
100
|
+
|
|
101
|
+
# Use git log to get commit timestamps and affected files
|
|
102
|
+
# Format: timestamp\n<blank>\nfile1\nfile2\n...\n<blank>\n
|
|
103
|
+
# Note: This may raise git.GitCommandError if git log fails
|
|
104
|
+
log_output = repo.git.log("--format=%ct", "--name-only", "--", *chunk)
|
|
105
|
+
|
|
106
|
+
# Parse the output
|
|
107
|
+
# Format is: timestamp\n<blank>\nfile1\nfile2\n...\n<blank>\n
|
|
108
|
+
lines = log_output.split("\n")
|
|
109
|
+
current_timestamp: int | None = None
|
|
110
|
+
file_timestamps: dict[str, int] = {}
|
|
111
|
+
|
|
112
|
+
for line in lines:
|
|
113
|
+
line = line.strip()
|
|
114
|
+
if not line:
|
|
115
|
+
# Blank line separates commits
|
|
116
|
+
continue
|
|
117
|
+
|
|
118
|
+
# Check if this is a timestamp (all digits)
|
|
119
|
+
if line.isdigit():
|
|
120
|
+
current_timestamp = int(line)
|
|
121
|
+
elif current_timestamp is not None:
|
|
122
|
+
# This is a file path, normalize it
|
|
123
|
+
file_path_normalized = line.replace("\\", "/")
|
|
124
|
+
if file_path_normalized in chunk_set:
|
|
125
|
+
# Track the most recent (largest) timestamp for each file
|
|
126
|
+
if (
|
|
127
|
+
file_path_normalized not in file_timestamps
|
|
128
|
+
or file_timestamps[file_path_normalized] < current_timestamp
|
|
129
|
+
):
|
|
130
|
+
file_timestamps[file_path_normalized] = current_timestamp
|
|
131
|
+
|
|
132
|
+
# Convert timestamps to datetime and map back to original PurePath objects
|
|
133
|
+
for rel_path in chunk:
|
|
134
|
+
original_path = rel_path_to_original[rel_path]
|
|
135
|
+
if rel_path in file_timestamps:
|
|
136
|
+
result[original_path] = datetime.fromtimestamp(
|
|
137
|
+
file_timestamps[rel_path]
|
|
138
|
+
)
|
|
139
|
+
else:
|
|
140
|
+
# File has no git history
|
|
141
|
+
result[original_path] = datetime.now()
|
|
142
|
+
|
|
143
|
+
return result
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _apply_last_modified_map(
|
|
147
|
+
resource: Resource, last_modified_cache: dict[PurePath, datetime]
|
|
148
|
+
) -> None:
|
|
149
|
+
"""
|
|
150
|
+
Recursively apply last_modified dates from cache to a Resource tree.
|
|
151
|
+
|
|
152
|
+
Updates file resources with dates from the cache, then updates directory
|
|
153
|
+
resources to be the max of their children.
|
|
154
|
+
"""
|
|
155
|
+
# Apply to children first (depth-first)
|
|
156
|
+
for child in resource.children:
|
|
157
|
+
_apply_last_modified_map(child, last_modified_cache)
|
|
158
|
+
|
|
159
|
+
# Apply to this resource if it's a file
|
|
160
|
+
if resource.type == "file" and resource.src_path in last_modified_cache:
|
|
161
|
+
resource.last_modified = last_modified_cache[resource.src_path]
|
|
162
|
+
|
|
163
|
+
# Update directory resources to be max of children
|
|
164
|
+
if resource.children:
|
|
165
|
+
resource.last_modified = max(child.last_modified for child in resource.children)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _get_resources(
|
|
169
|
+
dir: Dir,
|
|
170
|
+
parent_path: PurePath,
|
|
171
|
+
processors: list[Processor],
|
|
172
|
+
repo: git.Repo,
|
|
173
|
+
file_paths: list[PurePath] | None = None,
|
|
174
|
+
) -> list[Resource]:
|
|
175
|
+
child_resources: list[Resource] = []
|
|
176
|
+
dir_resource: Resource | None = None
|
|
177
|
+
|
|
178
|
+
# do a first pass to see if we can find a "dir_resource"
|
|
179
|
+
for file in dir.files:
|
|
180
|
+
for processor in processors:
|
|
181
|
+
if processor.pattern.search(file.name):
|
|
182
|
+
filetype, metadata, _ = processor.process(file.content or "")
|
|
183
|
+
if filetype != "file":
|
|
184
|
+
if dir_resource is None:
|
|
185
|
+
dir_resource = Resource(
|
|
186
|
+
name=dir.path.name,
|
|
187
|
+
src_path=dir.path,
|
|
188
|
+
path=PurePath(),
|
|
189
|
+
type=filetype,
|
|
190
|
+
metadata=metadata,
|
|
191
|
+
content=None,
|
|
192
|
+
children=[],
|
|
193
|
+
last_modified=datetime.now(),
|
|
194
|
+
)
|
|
195
|
+
# parent path is now the dir_resource
|
|
196
|
+
parent_path = dir_resource.path
|
|
197
|
+
else:
|
|
198
|
+
# a previous processor picked this up, augment the metadata
|
|
199
|
+
# FIXME: I think we want to be able to override the name as well
|
|
200
|
+
dir_resource.metadata.update(metadata)
|
|
201
|
+
|
|
202
|
+
# do a second pass to find all the child resources, placing
|
|
203
|
+
# their paths relative to the parent
|
|
204
|
+
for file in dir.files:
|
|
205
|
+
for processor in processors:
|
|
206
|
+
if processor.pattern.search(file.name):
|
|
207
|
+
_, metadata, _ = processor.process(file.content or "")
|
|
208
|
+
# Collect file path for batch querying
|
|
209
|
+
if file_paths is not None:
|
|
210
|
+
file_paths.append(file.src_path)
|
|
211
|
+
|
|
212
|
+
child_resources.append(
|
|
213
|
+
Resource(
|
|
214
|
+
name=file.name,
|
|
215
|
+
path=parent_path / file.name,
|
|
216
|
+
src_path=file.src_path,
|
|
217
|
+
type="file",
|
|
218
|
+
metadata=metadata,
|
|
219
|
+
content=file.content,
|
|
220
|
+
children=[],
|
|
221
|
+
last_modified=datetime.now(),
|
|
222
|
+
)
|
|
223
|
+
)
|
|
224
|
+
# Will only match once!
|
|
225
|
+
break
|
|
226
|
+
|
|
227
|
+
for subdir in dir.dirs:
|
|
228
|
+
child_resources.extend(
|
|
229
|
+
_get_resources(
|
|
230
|
+
subdir,
|
|
231
|
+
parent_path / subdir.path.name,
|
|
232
|
+
processors,
|
|
233
|
+
repo,
|
|
234
|
+
file_paths,
|
|
235
|
+
)
|
|
236
|
+
)
|
|
237
|
+
|
|
238
|
+
# if this directory did not define a new resource, append any newly
|
|
239
|
+
# found resources to the parent
|
|
240
|
+
if dir_resource is None:
|
|
241
|
+
return child_resources
|
|
242
|
+
|
|
243
|
+
# otherwise, add the child resources to the dir_resource, and return that
|
|
244
|
+
dir_resource.children = child_resources
|
|
245
|
+
|
|
246
|
+
# Update dir_resource last_modified to be the most recent from all children
|
|
247
|
+
if child_resources:
|
|
248
|
+
dir_resource.last_modified = max(
|
|
249
|
+
child.last_modified for child in child_resources
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
return [dir_resource]
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def get_resources(repo: git.Repo, dir: Dir, processors: list[Processor]) -> Resource:
|
|
256
|
+
"""
|
|
257
|
+
Gets a list of resources from a scanned filesystem.
|
|
258
|
+
|
|
259
|
+
Will process each file in the filesystem with the provided processors,
|
|
260
|
+
note that order matters: the first processor that matches a given file
|
|
261
|
+
will be used and no further processors will be applied after that.
|
|
262
|
+
"""
|
|
263
|
+
# Collect file paths as we create resources (single pass)
|
|
264
|
+
file_paths: list[PurePath] = []
|
|
265
|
+
children = _get_resources(dir, PurePath(), processors, repo, file_paths)
|
|
266
|
+
|
|
267
|
+
# Batch query all commit dates at once
|
|
268
|
+
scan_root = dir.path
|
|
269
|
+
last_modified_map = _get_last_modified_batch(repo, file_paths, scan_root)
|
|
270
|
+
|
|
271
|
+
root_resource = Resource(
|
|
272
|
+
name=dir.path.name,
|
|
273
|
+
path=PurePath(),
|
|
274
|
+
src_path=dir.path,
|
|
275
|
+
type="repo",
|
|
276
|
+
metadata={},
|
|
277
|
+
children=children,
|
|
278
|
+
content=None,
|
|
279
|
+
last_modified=datetime.now(),
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
# Apply last_modified dates
|
|
283
|
+
_apply_last_modified_map(root_resource, last_modified_map)
|
|
284
|
+
|
|
285
|
+
return root_resource
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
<!--
|
|
2
|
+
Commented sections (like this) are meant to be explanatory. Feel free to delete them.
|
|
3
|
+
|
|
4
|
+
To create IDRs without these comments:
|
|
5
|
+
- Use the --no-comments flag: rp idr new "Title" --no-comments
|
|
6
|
+
- Or set the environment variable: export RP_IDR_NO_COMMENTS=1
|
|
7
|
+
-->
|
|
8
|
+
|
|
9
|
+
# {{ title }}
|
|
10
|
+
|
|
11
|
+
Owner: {{ author }}
|
|
12
|
+
|
|
13
|
+
## Overview
|
|
14
|
+
|
|
15
|
+
### Problem Statement
|
|
16
|
+
|
|
17
|
+
<!--
|
|
18
|
+
* 3-5 lines
|
|
19
|
+
* Statement of the goal
|
|
20
|
+
* Help people decide if they want to invest in reading this on a skim
|
|
21
|
+
-->
|
|
22
|
+
|
|
23
|
+
### Context (as needed)
|
|
24
|
+
|
|
25
|
+
### Goals
|
|
26
|
+
|
|
27
|
+
<!--
|
|
28
|
+
* Describe major goals of this implementation
|
|
29
|
+
* Does not need to be exhaustive, less is more
|
|
30
|
+
-->
|
|
31
|
+
|
|
32
|
+
### Non-Goals
|
|
33
|
+
|
|
34
|
+
<!--
|
|
35
|
+
* Describe non-goals of this implementation
|
|
36
|
+
* Does not need to be exhaustive, less is more
|
|
37
|
+
-->
|
|
38
|
+
|
|
39
|
+
### Proposed Solution
|
|
40
|
+
|
|
41
|
+
<!--
|
|
42
|
+
Describe in relatively short form (ideally <200 words) the proposed solution
|
|
43
|
+
-->
|
|
44
|
+
|
|
45
|
+
## Detailed Design (as needed)
|
|
46
|
+
|
|
47
|
+
<!--
|
|
48
|
+
Go into more detail on the design if needed. Totally optional. Note there
|
|
49
|
+
is a section below for describing the exact shape of the implementation.
|
|
50
|
+
-->
|
|
51
|
+
|
|
52
|
+
## Rollout plan (as needed)
|
|
53
|
+
|
|
54
|
+
<!--
|
|
55
|
+
Describe how this feature will be rolled out, especially if there are breaking
|
|
56
|
+
changes required
|
|
57
|
+
-->
|
|
58
|
+
|
|
59
|
+
## Cross cutting concerns (as needed)
|
|
60
|
+
|
|
61
|
+
<!--
|
|
62
|
+
Describe the design from other angles: security, reliability, etc.
|
|
63
|
+
-->
|
|
64
|
+
|
|
65
|
+
## Alternatives considered (as needed)
|
|
66
|
+
|
|
67
|
+
<!--
|
|
68
|
+
Describe alternatives to this design that were considered. Doesn't need to
|
|
69
|
+
be exhaustive, only add something here if you think someone is likely to ask
|
|
70
|
+
about it.
|
|
71
|
+
-->
|
|
72
|
+
|
|
73
|
+
## Future plans (as needed)
|
|
74
|
+
|
|
75
|
+
<!--
|
|
76
|
+
Things that may be added in the future
|
|
77
|
+
-->
|
|
78
|
+
|
|
79
|
+
## Other reading (as needed)
|
|
80
|
+
|
|
81
|
+
<!--
|
|
82
|
+
Provide links to prior art
|
|
83
|
+
-->
|
|
84
|
+
|
|
85
|
+
## Implementation (ephemeral)
|
|
86
|
+
|
|
87
|
+
<!--
|
|
88
|
+
Details on the implementation of this design. Can include checklists,
|
|
89
|
+
detailed descriptions of code changes, etc. Whatever is most helpful
|
|
90
|
+
to the humans and/or computers responsible for implementation!
|
|
91
|
+
-->
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: repo-parser
|
|
3
|
+
Version: 0.0.3
|
|
4
|
+
Summary: A library for extracting metadata out of a source repository.
|
|
5
|
+
Author-email: William Lachance <wlach@protonmail.com>
|
|
6
|
+
Requires-Python: >=3.11
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: GitPython
|
|
11
|
+
Requires-Dist: jinja2
|
|
12
|
+
Requires-Dist: python-slugify
|
|
13
|
+
Requires-Dist: typer
|
|
14
|
+
Project-URL: Home, https://github.com/wlach/repo-parser
|
|
15
|
+
|
|
16
|
+
# repo-parser
|
|
17
|
+
|
|
18
|
+
This is a set of python scripts and tools for extracting metadata and structure
|
|
19
|
+
out of a monorepo, for the purposes of generating a service registry, unified documentation, or other types of data out of the contents. It also contains some
|
|
20
|
+
tooling designed around a new concept called "Implementation Decision Records" (idrs), designed to aid both implementation (especially where assisted by LLMs) and later code archaeology efforts.
|
|
21
|
+
|
|
22
|
+
It should be considered an experiment, rather than production software. It is inspired by my experiences in the software industry and being frustrated with current solutions being some combination of:
|
|
23
|
+
|
|
24
|
+
- Time consuming to configure and maintain
|
|
25
|
+
- Requiring the specification of redundant metadata (which is bound to get out of date)
|
|
26
|
+
|
|
27
|
+
Current assumptions:
|
|
28
|
+
|
|
29
|
+
- The files you want to process fit into memory
|
|
30
|
+
- You don't care about history (repo-parser gives a snapshot in time)
|
|
31
|
+
|
|
32
|
+
You can see a demo of a documentation site generated from this repo at:
|
|
33
|
+
|
|
34
|
+
https://repo-parser-demo.netlify.app/
|
|
35
|
+
|
|
36
|
+
> [!NOTE]
|
|
37
|
+
> repo-parser is partly created with LLMs (specifically, Claude Code and ChatGPT),
|
|
38
|
+
> with heavy human curation.
|
|
39
|
+
|
|
40
|
+
## Usage
|
|
41
|
+
|
|
42
|
+
### Creating IDRs (Implementation Decision Records)
|
|
43
|
+
|
|
44
|
+
repo-parser includes a CLI tool (`rp`) for creating Implementation Decision Records (IDRs). IDRs are lightweight documents that capture both the intent and implementation details of code changes, designed to be committed alongside your code.
|
|
45
|
+
|
|
46
|
+
They are intended to be useful for both humans and computational agents (LLMs).
|
|
47
|
+
|
|
48
|
+
**Why IDRs?** Traditional design docs are too heavyweight for implementation. ADRs are great for decisions but lack implementation detail. IDRs bridge the gap—detailed enough to guide implementation, ephemeral enough to not become stale documentation.
|
|
49
|
+
|
|
50
|
+
Create a new IDR:
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
rp idr new "Your Feature Title"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
This will create a new markdown file in the `idrs/` directory with:
|
|
57
|
+
|
|
58
|
+
- A UTC timestamp prefix (e.g., `202512301600`)
|
|
59
|
+
- A slugified version of your title
|
|
60
|
+
- Pre-filled template with your git author name
|
|
61
|
+
|
|
62
|
+
Example:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
rp idr new "Add last modified metadata"
|
|
66
|
+
# Creates: idrs/202512301600-add-last-modified-metadata.md
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
The IDR template includes sections for:
|
|
70
|
+
|
|
71
|
+
- Problem Statement
|
|
72
|
+
- Context
|
|
73
|
+
- Goals and Non-Goals
|
|
74
|
+
- Proposed Solution
|
|
75
|
+
- Detailed Design
|
|
76
|
+
- Implementation notes (ephemeral)
|
|
77
|
+
|
|
78
|
+
This feature was itself implemented using an IDR—see [idrs/202512301631-idrs.md](idrs/202512301631-idrs.md) for more about the IDR format and philosophy.
|
|
79
|
+
|
|
80
|
+
### Using repo-parser as a Library
|
|
81
|
+
|
|
82
|
+
For documentation generation and metadata extraction, see the example in `example/example_parser.py`.
|
|
83
|
+
|
|
84
|
+
An early version of repo-parser is available on pypi as https://pypi.org/project/repo-parser. Use at your own risk!
|
|
85
|
+
|
|
86
|
+
## Local development
|
|
87
|
+
|
|
88
|
+
You can experiment with the local demo by running `make example`.
|
|
89
|
+
It should live-reload as you make changes in the `example/repo` directory.
|
|
90
|
+
|
|
91
|
+
For other tasks, look at the (very simple) `Makefile` in the root directory.
|
|
92
|
+
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
repo_parser/__init__.py,sha256=JRPNDVsbAMBf_yiRP6Xm13kpnJEvYhqMph-OFem67P8,283
|
|
2
|
+
repo_parser/cli.py,sha256=ze4vBjiF8LriRS0qFSx1IN6iWyC4h50tZzrzE4jYgxg,4596
|
|
3
|
+
repo_parser/filesystem.py,sha256=q8ZiTvp2y9lhJdbES6drMVQhORMd1j0AUEKMGnS3Sas,3602
|
|
4
|
+
repo_parser/processor.py,sha256=O5AZisIY5GzX0SBCMuGeWiDtnNd5_nDHVmZuhwn-82Q,254
|
|
5
|
+
repo_parser/resource.py,sha256=jmLFI1VPBnHHDxGp9gqe9niKr6h7eHM68cwiNhyya3M,10607
|
|
6
|
+
repo_parser/templates/idr.md,sha256=fB7RbKwkZnu8BHJRYEoSejVYcpvDlGRBR5dJtmD3UGg,1854
|
|
7
|
+
repo_parser-0.0.3.dist-info/entry_points.txt,sha256=jH0jHkd3zXSJPZObgmTp8sMplnqfd31VN2VnINnYsQE,42
|
|
8
|
+
repo_parser-0.0.3.dist-info/licenses/LICENSE,sha256=HpJ9NsyYs74vwDiFpunj0wdxKEOS74VFNJb7E2SdZis,1088
|
|
9
|
+
repo_parser-0.0.3.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82
|
|
10
|
+
repo_parser-0.0.3.dist-info/METADATA,sha256=PeeQE6BzRcpVfweS9ilQMhlsUEJiEjkbFyw-JrUsxWg,3484
|
|
11
|
+
repo_parser-0.0.3.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023-2024 William Lachance
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|