repo-parser 0.0.1__tar.gz

repo_parser-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023-2024 William Lachance
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

repo_parser-0.0.1/PKG-INFO

@@ -0,0 +1,37 @@
+Metadata-Version: 2.1
+Name: repo-parser
+Version: 0.0.1
+Summary: A library for extracting metadata out of a source repository.
+Author-email: William Lachance <wlach@protonmail.com>
+Description-Content-Type: text/markdown
+Classifier: License :: OSI Approved :: MIT License
+Requires-Dist: GitPython
+Project-URL: Home, https://github.com/wlach/repo-parser
+
+# repo-parser
+
+This is a set of python scripts and tools for extracting metadata and structure
+out of a monorepo, for the purposes of generating a service registry, unified documentation, or other types of data out of the contents.
+
+It should be considered an experiment, rather than production software. It is inspired by my experiences in the software industry and being frustrated with current solutions being some combination of:
+
+- Expensive
+- Time consuming to configure and maintain
+- Requiring the specification of redundant metadata (which is bound to get out of date)
+
+Assumptions:
+
+- The files you want to process fit into memory
+- You don't care about history
+
+You can see a demo of a documentation site generated from this repo at:
+
+https://repo-parser-demo.netlify.app/
+
+## Local development
+
+You can experiment with the local demo by running `make example`.
+It should live-reload as you make changes in the `example/repo` directory.
+
+For other tasks, look at the (very simple) `Makefile` in the root directory.
+

repo_parser-0.0.1/README.md

@@ -0,0 +1,26 @@
+# repo-parser
+
+This is a set of python scripts and tools for extracting metadata and structure
+out of a monorepo, for the purposes of generating a service registry, unified documentation, or other types of data out of the contents.
+
+It should be considered an experiment, rather than production software. It is inspired by my experiences in the software industry and being frustrated with current solutions being some combination of:
+
+- Expensive
+- Time consuming to configure and maintain
+- Requiring the specification of redundant metadata (which is bound to get out of date)
+
+Assumptions:
+
+- The files you want to process fit into memory
+- You don't care about history
+
+You can see a demo of a documentation site generated from this repo at:
+
+https://repo-parser-demo.netlify.app/
+
+## Local development
+
+You can experiment with the local demo by running `make example`.
+It should live-reload as you make changes in the `example/repo` directory.
+
+For other tasks, look at the (very simple) `Makefile` in the root directory.

repo_parser-0.0.1/pyproject.toml

@@ -0,0 +1,18 @@
+[build-system]
+requires = ["flit_core >=3.2,<4"]
+build-backend = "flit_core.buildapi"
+
+[project]
+name = "repo-parser"
+authors = [{ name = "William Lachance", email = "wlach@protonmail.com" }]
+readme = "README.md"
+license = { file = "LICENSE" }
+classifiers = ["License :: OSI Approved :: MIT License"]
+dynamic = ["version", "description"]
+dependencies = ["GitPython"]
+
+[project.urls]
+Home = "https://github.com/wlach/repo-parser"
+
+[tool.ruff.lint]
+select = ["E4", "E7", "E9", "F", "B", "I"]

repo_parser-0.0.1/repo_parser/__init__.py

@@ -0,0 +1,11 @@
+"""
+A library for extracting metadata out of a source repository.
+"""
+
+__version__ = "0.0.1"
+
+from .filesystem import scan
+from .processor import DEFAULT_PROCESSORS
+from .resource import Resource, get_resources
+
+__all__ = ["scan", "get_resources", "Resource", "DEFAULT_PROCESSORS"]

repo_parser-0.0.1/repo_parser/filesystem.py

@@ -0,0 +1,70 @@
+import pathlib
+from dataclasses import dataclass
+from pathlib import PurePath
+from typing import List, Optional
+
+import git
+
+from .processor import Processor
+
+
+@dataclass
+class File:
+    name: str
+    src_path: PurePath
+    content: Optional[str]
+
+
+@dataclass
+class Dir:
+    path: pathlib.Path
+    files: List[File]
+    dirs: List["Dir"]
+
+
+def _scan(path: pathlib.Path, processors: List[Processor], repo: git.Repo) -> Dir:
+    dir = Dir(path=pathlib.PurePath(path), files=[], dirs=[])
+
+    entries = [entry for entry in path.iterdir()]
+    if not entries:
+        return dir
+    # FIXME: At some point we should only ignore .git in the root directory
+    ignored = set(repo.ignored(*[entry.name for entry in entries]) + [".git"])
+
+    for entry in entries:
+        if entry.name in ignored:
+            continue
+        elif entry.is_file():
+            for processor in processors:
+                if processor.pattern.search(str(entry)):
+                    content = entry.read_text() if processor.read_content else None
+                    dir.files.append(
+                        File(
+                            name=entry.name,
+                            src_path=(path / entry.name),
+                            content=content,
+                        )
+                    )
+        elif entry.is_dir():
+            dir.dirs.append(_scan(path / entry.name, processors, repo))
+
+    return dir
+
+
+
+def scan(path: pathlib.Path, processors: List[Processor]) -> Dir:
+    """
+    Scans a GitHub repository for files and subdirectories, returning a data
+    structure representing the directory tree.
+    
+    Takes in a list of processors to figure out which files should be included
+    in the returned tree.
+
+    Under the hood, this uses the `git` library to scan the repository, skipping
+    files in .gitignore.
+
+    This step does not do any post-processing of the files, though their
+    content is read in if one of their processors requires it.
+    """
+    repo = git.Repo(path)
+    return _scan(path, processors, repo)

repo_parser-0.0.1/repo_parser/processor.py

@@ -0,0 +1,25 @@
+import re
+from dataclasses import dataclass
+
+import frontmatter
+
+
+@dataclass
+class Processor:
+    pattern: re.Pattern
+    process: callable
+    read_content: bool
+
+
+def _parse_markdown(file_contents: str):
+    metadata, _ = frontmatter.parse(file_contents)
+
+    filetype = metadata.get("type", "file")
+    metadata.pop("type", None)  # remove "type" from the metadata
+
+    return filetype, metadata, file_contents
+
+
+DEFAULT_PROCESSORS = [
+    Processor(re.compile("\.md$"), _parse_markdown, True),
+]

repo_parser-0.0.1/repo_parser/resource.py

@@ -0,0 +1,89 @@
+from dataclasses import dataclass
+from pathlib import PurePath
+from typing import List, Optional
+
+from .filesystem import Dir
+from .processor import Processor
+
+
+@dataclass
+class Resource:
+    name: str
+    src_path: PurePath
+    path: PurePath
+    type: str
+    metadata: dict
+    content: Optional[str]
+    children: List["Resource"]
+
+
+def _get_resources(
+    dir: Dir, parent_path: PurePath, processors: List[Processor]
+) -> List[Resource]:
+    child_resources: List[Resource] = []
+    dir_resource: Optional[Resource] = None
+
+    # do a first pass to see if we can find a "dir_resource"
+    for file in dir.files:
+        for processor in processors:
+            if processor.pattern.search(file.name):
+                filetype, metadata, content = processor.process(file.content)
+                if filetype != "file":
+                    if dir_resource is None:
+                        dir_resource = Resource(
+                            name=dir.path.name,
+                            src_path=dir.path,
+                            path=PurePath(),
+                            type=filetype,
+                            metadata=metadata,
+                            content=None,
+                            children=[],
+                        )
+                        # parent path is now the dir_resource
+                        parent_path = dir_resource.path
+                    else:
+                        # a previous processor picked this up, augment the metadata
+                        # FIXME: I think we want to be able to override the name as well
+                        dir_resource.metadata.update(metadata)
+
+    # do a second pass to find all the child resources, placing
+    # their paths relative to the parent
+    for file in dir.files:
+        if processor.pattern.search(file.name):
+            child_resources.append(
+                Resource(
+                    name=file.name,
+                    path=parent_path / file.name,
+                    src_path=file.src_path,
+                    type="file",
+                    metadata=metadata,
+                    content=file.content,
+                    children=[],
+                )
+            )
+
+    for subdir in dir.dirs:
+        child_resources.extend(
+            _get_resources(subdir, parent_path / subdir.path.name, processors)
+        )
+
+    # if this directory did not define a new resource, append any newly
+    # found resources to the parent
+    if dir_resource is None:
+        return child_resources
+
+    # otherwise, add the child resources to the dir_resource, and return that
+    dir_resource.children = child_resources
+    return [dir_resource]
+
+
+def get_resources(dir: Dir, processors: List[Processor]) -> Resource:
+    return Resource(
+        name=dir.path.name,
+        path=PurePath(),
+        src_path=dir.path,
+        type="repo",
+        metadata={},
+        children=_get_resources(dir, PurePath(), processors),
+        content=None,
+    )