code-maat-python 0.1.0__py3-none-any.whl

code_maat_python/parser.py

@@ -0,0 +1,232 @@
+"""Git log parser for code-maat-python.
+
+Parses git log output into pandas DataFrames for analysis.
+"""
+
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Iterator, TextIO
+
+import pandas as pd
+import re
+
+
+@dataclass
+class CommitRecord:
+    """Represents a single commit with file changes."""
+
+    rev: str
+    date: str
+    author: str
+    entity: str
+    loc_added: int
+    loc_deleted: int
+
+
+class GitLogSchema:
+    """Defines the DataFrame schema for parsed git logs."""
+
+    # Column names
+    REV = "rev"
+    DATE = "date"
+    AUTHOR = "author"
+    ENTITY = "entity"
+    LOC_ADDED = "loc_added"
+    LOC_DELETED = "loc_deleted"
+
+    # Column types for optimization
+    DTYPES = {
+        REV: "category",
+        AUTHOR: "category",
+        ENTITY: "category",
+        LOC_ADDED: "int32",
+        LOC_DELETED: "int32",
+    }
+
+    @classmethod
+    def columns(cls) -> list[str]:
+        """Return ordered list of column names."""
+        return [cls.REV, cls.DATE, cls.AUTHOR, cls.ENTITY, cls.LOC_ADDED, cls.LOC_DELETED]
+
+    @classmethod
+    def create_empty_dataframe(cls) -> pd.DataFrame:
+        """Create an empty DataFrame with the correct schema."""
+        df = pd.DataFrame(columns=cls.columns())
+        df[cls.DATE] = pd.to_datetime(df[cls.DATE])
+        return df
+
+
+class GitLogParser:
+    """Parser for git log output.
+
+    Parses output from:
+        git log --all -M -C --numstat --date=short --pretty=format:'--%h--%cd--%cn'
+    """
+
+    # Regex patterns
+    COMMIT_PATTERN = re.compile(r"^--([a-f0-9]+)--(\d{4}-\d{2}-\d{2})--(.*)$")
+    CHANGE_PATTERN = re.compile(r"^(\d+|-)\t(\d+|-)\t(.+)$")
+
+    def __init__(self) -> None:
+        """Initialize the parser."""
+        self._current_commit: dict[str, str] | None = None
+        self._records: list[CommitRecord] = []
+
+    def parse_file(self, filepath: Path | str) -> pd.DataFrame:
+        """Parse a git log file.
+
+        Args:
+            filepath: Path to the git log file
+
+        Returns:
+            DataFrame with columns: rev, date, author, entity, loc_added, loc_deleted
+
+        Raises:
+            FileNotFoundError: If the file doesn't exist
+            ValueError: If the file format is invalid
+        """
+        filepath = Path(filepath)
+        if not filepath.exists():
+            raise FileNotFoundError(f"Git log file not found: {filepath}")
+
+        with open(filepath, "r", encoding="utf-8") as f:
+            return self.parse_stream(f)
+
+    def parse_stream(self, stream: TextIO) -> pd.DataFrame:
+        """Parse a git log from a stream.
+
+        Args:
+            stream: Text stream (file object) containing git log
+
+        Returns:
+            DataFrame with parsed commit data
+        """
+        self._current_commit = None
+        self._records = []
+
+        for line_num, line in enumerate(stream, start=1):
+            line = line.strip()
+
+            # Skip empty lines
+            if not line:
+                continue
+
+            # Try to parse as commit header
+            if self._try_parse_commit(line):
+                continue
+
+            # Try to parse as file change
+            if self._try_parse_change(line, line_num):
+                continue
+
+            # If we get here, the line didn't match any pattern
+            # This might be okay (e.g., continuation of author name with newline)
+            # But we should warn if it looks like data
+            if line.strip() and not line.startswith(" "):
+                print(f"Warning: Unparsed line {line_num}: {line[:50]}")
+
+        return self._to_dataframe()
+
+    def _try_parse_commit(self, line: str) -> bool:
+        """Try to parse line as commit header.
+
+        Args:
+            line: Line from git log
+
+        Returns:
+            True if line was a commit header, False otherwise
+        """
+        match = self.COMMIT_PATTERN.match(line)
+        if match:
+            self._current_commit = {
+                "rev": match.group(1),
+                "date": match.group(2),
+                "author": match.group(3).strip(),
+            }
+            return True
+        return False
+
+    def _try_parse_change(self, line: str, line_num: int) -> bool:
+        """Try to parse line as file change.
+
+        Args:
+            line: Line from git log
+            line_num: Line number (for error reporting)
+
+        Returns:
+            True if line was a file change, False otherwise
+        """
+        match = self.CHANGE_PATTERN.match(line)
+        if match and self._current_commit:
+            # Parse added/deleted, handling binary files (-)
+            loc_added = 0 if match.group(1) == "-" else int(match.group(1))
+            loc_deleted = 0 if match.group(2) == "-" else int(match.group(2))
+            entity = match.group(3)
+
+            record = CommitRecord(
+                rev=self._current_commit["rev"],
+                date=self._current_commit["date"],
+                author=self._current_commit["author"],
+                entity=entity,
+                loc_added=loc_added,
+                loc_deleted=loc_deleted,
+            )
+            self._records.append(record)
+            return True
+
+        if match and not self._current_commit:
+            print(f"Warning: File change without commit header at line {line_num}")
+            return False
+
+        return False
+
+    def _to_dataframe(self) -> pd.DataFrame:
+        """Convert parsed records to DataFrame.
+
+        Returns:
+            DataFrame with proper schema and types
+        """
+        if not self._records:
+            return GitLogSchema.create_empty_dataframe()
+
+        # Convert records to dict
+        data = {
+            GitLogSchema.REV: [r.rev for r in self._records],
+            GitLogSchema.DATE: [r.date for r in self._records],
+            GitLogSchema.AUTHOR: [r.author for r in self._records],
+            GitLogSchema.ENTITY: [r.entity for r in self._records],
+            GitLogSchema.LOC_ADDED: [r.loc_added for r in self._records],
+            GitLogSchema.LOC_DELETED: [r.loc_deleted for r in self._records],
+        }
+
+        df = pd.DataFrame(data)
+
+        # Convert date to datetime
+        df[GitLogSchema.DATE] = pd.to_datetime(df[GitLogSchema.DATE])
+
+        # Apply categorical types for memory efficiency
+        df[GitLogSchema.REV] = df[GitLogSchema.REV].astype("category")
+        df[GitLogSchema.AUTHOR] = df[GitLogSchema.AUTHOR].astype("category")
+        df[GitLogSchema.ENTITY] = df[GitLogSchema.ENTITY].astype("category")
+
+        return df
+
+
+# Convenience function
+def parse_git_log(filepath: Path | str) -> pd.DataFrame:
+    """Parse a git log file.
+
+    Convenience function for quick parsing.
+
+    Args:
+        filepath: Path to git log file
+
+    Returns:
+        DataFrame with parsed commit data
+
+    Example:
+        >>> df = parse_git_log("git.log")
+        >>> print(df.head())
+    """
+    parser = GitLogParser()
+    return parser.parse_file(filepath)

code_maat_python/pipeline.py

@@ -0,0 +1,112 @@
+"""Data pipeline utilities for code-maat-python.
+
+Provides utilities for transforming and filtering DataFrames between
+parsing and analysis stages.
+"""
+
+import pandas as pd
+from pathlib import Path
+from typing import Callable
+
+from code_maat_python.parser import parse_git_log, GitLogSchema
+
+
+def load_git_log(filepath: Path | str) -> pd.DataFrame:
+    """Load and parse a git log file.
+
+    Convenience function that wraps the parser.
+
+    Args:
+        filepath: Path to git log file
+
+    Returns:
+        Parsed DataFrame
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+    """
+    return parse_git_log(filepath)
+
+
+def filter_by_date(
+    df: pd.DataFrame, start_date: str | None = None, end_date: str | None = None
+) -> pd.DataFrame:
+    """Filter DataFrame by date range.
+
+    Args:
+        df: Input DataFrame
+        start_date: Start date (YYYY-MM-DD) or None for no start limit
+        end_date: End date (YYYY-MM-DD) or None for no end limit
+
+    Returns:
+        Filtered DataFrame
+
+    Example:
+        >>> df_filtered = filter_by_date(df, start_date="2023-01-01")
+    """
+    result = df.copy()
+
+    if start_date:
+        start = pd.to_datetime(start_date)
+        result = result[result[GitLogSchema.DATE] >= start]
+
+    if end_date:
+        end = pd.to_datetime(end_date)
+        result = result[result[GitLogSchema.DATE] <= end]
+
+    return result
+
+
+def filter_by_entity_pattern(df: pd.DataFrame, pattern: str) -> pd.DataFrame:
+    """Filter DataFrame by entity path pattern.
+
+    Args:
+        df: Input DataFrame
+        pattern: Regex pattern to match against entity paths
+
+    Returns:
+        Filtered DataFrame
+
+    Example:
+        >>> df_py = filter_by_entity_pattern(df, r".*\\.py$")
+    """
+    mask = df[GitLogSchema.ENTITY].str.contains(pattern, na=False, regex=True)
+    return df[mask].copy()
+
+
+def apply_transformation(
+    df: pd.DataFrame, transform_fn: Callable[[pd.DataFrame], pd.DataFrame]
+) -> pd.DataFrame:
+    """Apply a transformation function to DataFrame.
+
+    Generic utility for applying transformations in pipeline.
+
+    Args:
+        df: Input DataFrame
+        transform_fn: Function that takes and returns a DataFrame
+
+    Returns:
+        Transformed DataFrame
+    """
+    return transform_fn(df)
+
+
+def validate_dataframe(df: pd.DataFrame) -> None:
+    """Validate that DataFrame has required columns and types.
+
+    Args:
+        df: DataFrame to validate
+
+    Raises:
+        ValueError: If DataFrame is invalid
+    """
+    required_cols = set(GitLogSchema.columns())
+    actual_cols = set(df.columns)
+
+    if not required_cols.issubset(actual_cols):
+        missing = required_cols - actual_cols
+        raise ValueError(f"DataFrame missing required columns: {missing}")
+
+    # Check date column is datetime
+    if not pd.api.types.is_datetime64_any_dtype(df[GitLogSchema.DATE]):
+        raise ValueError(f"Column '{GitLogSchema.DATE}' must be datetime type")

code_maat_python/transformers/grouper.py

@@ -0,0 +1,204 @@
+r"""Architectural grouper for mapping entities to logical boundaries.
+
+This module provides functionality to map physical file paths to logical
+architectural groups (layers, components, subsystems) before analysis.
+
+Grouping is specified via text files with two pattern types:
+1. Plain text paths: "src/Features/Core => Core"
+   - Automatically converted to regex: ^src/Features/Core/
+2. Explicit regex: r"^src\/.*Test.*$ => Tests"
+   - Used as-is for flexible matching
+
+First matching pattern wins (order matters). Unmapped entities are filtered out.
+"""
+
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from re import Pattern
+
+import pandas as pd
+
+from code_maat_python.parser import GitLogSchema
+
+
+@dataclass
+class GroupPattern:
+    """A compiled grouping pattern.
+
+    Attributes:
+        pattern: Compiled regex pattern to match entity paths
+        logical_name: Logical group name to map to
+    """
+
+    pattern: Pattern[str]
+    logical_name: str
+
+
+def parse_group_specification(spec: str) -> list[GroupPattern]:
+    """Parse a grouping specification into compiled patterns.
+
+    The specification format is:
+        path_pattern => logical_name
+
+    Where path_pattern can be:
+    - Plain text path (e.g., "src/Core") - auto-converted to ^src/Core/
+    - Explicit regex (e.g., r"^src\\/.*Test.*$") - used as-is
+
+    Args:
+        spec: Multi-line string with grouping specifications
+
+    Returns:
+        List of compiled GroupPattern objects in specification order
+
+    Raises:
+        ValueError: If specification format is invalid or regex won't compile
+
+    Examples:
+        >>> spec = "src/Core => Core\\n^src\\\\/.*Test.*$ => Tests"
+        >>> patterns = parse_group_specification(spec)
+        >>> len(patterns)
+        2
+    """
+    patterns: list[GroupPattern] = []
+
+    for line_num, line in enumerate(spec.split("\n"), start=1):
+        # Skip empty lines and whitespace
+        line = line.strip()
+        if not line:
+            continue
+
+        # Check for separator
+        if "=>" not in line:
+            raise ValueError(
+                f"Invalid group specification at line {line_num}: "
+                f"Missing '=>' separator. Line: '{line}'"
+            )
+
+        # Split on separator
+        parts = line.split("=>", maxsplit=1)
+        if len(parts) != 2:
+            raise ValueError(
+                f"Invalid group specification at line {line_num}: "
+                f"Expected 'pattern => name' format. Line: '{line}'"
+            )
+
+        path_pattern = parts[0].strip()
+        logical_name = parts[1].strip()
+
+        if not path_pattern or not logical_name:
+            raise ValueError(
+                f"Invalid group specification at line {line_num}: "
+                f"Pattern and name cannot be empty. Line: '{line}'"
+            )
+
+        # Determine pattern type and compile
+        try:
+            if path_pattern.startswith("^"):
+                # Explicit regex pattern - use as-is
+                compiled_pattern = re.compile(path_pattern)
+            else:
+                # Plain text path - convert to regex with trailing /
+                # Escape special regex characters, then add ^ and /
+                escaped = re.escape(path_pattern)
+                regex_str = f"^{escaped}/"
+                compiled_pattern = re.compile(regex_str)
+        except re.error as e:
+            raise ValueError(
+                f"Invalid regex pattern at line {line_num}: {path_pattern}. " f"Error: {str(e)}"
+            ) from e
+
+        patterns.append(GroupPattern(pattern=compiled_pattern, logical_name=logical_name))
+
+    return patterns
+
+
+def map_entities_to_groups(df: pd.DataFrame, patterns: list[GroupPattern]) -> pd.DataFrame:
+    """Map entity names to logical groups based on patterns.
+
+    Applies patterns in order, using first match. Entities that don't match
+    any pattern are filtered out (not passed through).
+
+    Args:
+        df: DataFrame with GitLogSchema columns
+        patterns: List of compiled GroupPattern objects
+
+    Returns:
+        DataFrame with entity names replaced by logical group names.
+        Only includes rows where entity matched a pattern.
+
+    Examples:
+        >>> data = [{"entity": "src/Core/file.py", "author": "Alice", ...}]
+        >>> df = pd.DataFrame(data)
+        >>> spec = "src/Core => Core"
+        >>> patterns = parse_group_specification(spec)
+        >>> result = map_entities_to_groups(df, patterns)
+        >>> result.iloc[0]["entity"]
+        'Core'
+    """
+    # Handle empty inputs
+    if df.empty:
+        # Return empty DataFrame with correct schema
+        return GitLogSchema.create_empty_dataframe()
+
+    if not patterns:
+        # No patterns = filter out everything
+        empty_df = df.iloc[:0].copy()
+        return empty_df
+
+    # Map each entity to its logical group (or None if no match)
+    def find_logical_group(entity: str) -> str | None:
+        """Find the first matching logical group for an entity."""
+        for group_pattern in patterns:
+            if group_pattern.pattern.match(entity):
+                return group_pattern.logical_name
+        return None
+
+    # Apply mapping
+    entity_col = GitLogSchema.ENTITY
+    df_copy = df.copy()
+    df_copy["_logical_group"] = df_copy[entity_col].apply(find_logical_group)
+
+    # Filter out unmapped entities
+    df_filtered = df_copy[df_copy["_logical_group"].notna()].copy()
+
+    # Replace entity column with logical group
+    if not df_filtered.empty:
+        df_filtered[entity_col] = df_filtered["_logical_group"]
+
+    # Drop the temporary column
+    df_filtered = df_filtered.drop(columns=["_logical_group"])
+
+    return df_filtered
+
+
+def load_group_specification_file(filepath: Path | str) -> list[GroupPattern]:
+    """Load and parse a grouping specification file.
+
+    Convenience function that reads a file and parses its contents.
+
+    Args:
+        filepath: Path to grouping specification file
+
+    Returns:
+        List of compiled GroupPattern objects
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        ValueError: If file contents are invalid
+
+    Example:
+        >>> patterns = load_group_specification_file("layers.txt")
+        >>> len(patterns) > 0
+        True
+    """
+    filepath = Path(filepath)
+
+    if not filepath.exists():
+        raise FileNotFoundError(f"Group specification file not found: {filepath}")
+
+    if not filepath.is_file():
+        raise ValueError(f"Not a file: {filepath}")
+
+    spec = filepath.read_text(encoding="utf-8")
+    return parse_group_specification(spec)

code_maat_python/transformers/team_mapper.py

@@ -0,0 +1,132 @@
+"""Team mapper for aggregating author contributions to team level.
+
+This module provides functionality to map individual authors to teams, allowing
+analysis at the organizational team level rather than individual contributor level.
+
+Team mappings are specified via CSV files with format:
+    author,team
+    John Doe,Backend Team
+    Jane Smith,Backend Team
+
+Authors not in the mapping are preserved as-is (treated as individual "teams"),
+which allows detection of missing mappings rather than silent data loss.
+"""
+
+from io import StringIO
+from pathlib import Path
+
+import pandas as pd
+
+from code_maat_python.parser import GitLogSchema
+
+
+def parse_team_mapping_csv(csv_content: str) -> dict[str, str]:
+    """Parse a team mapping CSV into author->team dictionary.
+
+    The CSV format expected:
+        author,team
+        John Doe,Backend Team
+        Jane Smith,Frontend Team
+
+    Args:
+        csv_content: CSV string with author,team columns
+
+    Returns:
+        Dictionary mapping author names to team names
+
+    Raises:
+        ValueError: If CSV is missing required columns
+
+    Examples:
+        >>> csv = "author,team\\nAlice,Backend\\nBob,Frontend"
+        >>> mapping = parse_team_mapping_csv(csv)
+        >>> mapping["Alice"]
+        'Backend'
+    """
+    # Parse CSV (keep_default_na=False to preserve empty strings)
+    df = pd.read_csv(StringIO(csv_content), keep_default_na=False)
+
+    # Validate required columns
+    required_cols = {"author", "team"}
+    if not required_cols.issubset(set(df.columns)):
+        missing = required_cols - set(df.columns)
+        raise ValueError(
+            f"Missing required columns in team mapping CSV: {missing}. "
+            f"Expected columns: {required_cols}, got: {set(df.columns)}"
+        )
+
+    # Build author->team lookup dictionary
+    # If there are duplicates, last one wins (pandas default behavior)
+    # strict=False allows different lengths (though they should match in valid CSV)
+    mapping = dict(zip(df["author"], df["team"], strict=False))
+
+    return mapping
+
+
+def map_authors_to_teams(df: pd.DataFrame, team_mapping: dict[str, str]) -> pd.DataFrame:
+    """Map author names to team names based on mapping.
+
+    Authors not in the mapping are preserved as-is (not filtered out).
+    This allows detection of missing mappings.
+
+    Args:
+        df: DataFrame with GitLogSchema columns
+        team_mapping: Dictionary mapping author names to team names
+
+    Returns:
+        DataFrame with author column replaced by team names.
+        Authors not in mapping are kept unchanged.
+
+    Examples:
+        >>> data = [{"entity": "file.py", "author": "Alice", ...}]
+        >>> df = pd.DataFrame(data)
+        >>> mapping = {"Alice": "Backend Team"}
+        >>> result = map_authors_to_teams(df, mapping)
+        >>> result.iloc[0]["author"]
+        'Backend Team'
+    """
+    # Handle empty inputs
+    if df.empty:
+        return GitLogSchema.create_empty_dataframe()
+
+    # Make a copy to avoid modifying original
+    result = df.copy()
+
+    # Map authors to teams, preserving unmapped authors
+    # Using .get(author, author) keeps original if not in mapping
+    author_col = GitLogSchema.AUTHOR
+    result[author_col] = result[author_col].map(lambda author: team_mapping.get(author, author))
+
+    return result
+
+
+def load_team_mapping_file(filepath: Path | str) -> dict[str, str]:
+    """Load and parse a team mapping CSV file.
+
+    Convenience function that reads a file and parses its contents.
+
+    Args:
+        filepath: Path to team mapping CSV file
+
+    Returns:
+        Dictionary mapping author names to team names
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        ValueError: If file is invalid or missing required columns
+
+    Example:
+        >>> mapping = load_team_mapping_file("teams.csv")
+        >>> "Alice" in mapping
+        True
+    """
+    filepath = Path(filepath)
+
+    if not filepath.exists():
+        raise FileNotFoundError(f"Team mapping file not found: {filepath}")
+
+    if not filepath.is_file():
+        raise ValueError(f"Not a file: {filepath}")
+
+    csv_content = filepath.read_text(encoding="utf-8")
+    return parse_team_mapping_csv(csv_content)