codedd-cli 0.1.1__py3-none-any.whl

codedd_cli/scanner/file_walker.py

@@ -0,0 +1,213 @@
+"""
+Local repository scanner.
+
+Walks a Git repository directory, classifies each file, counts lines of code,
+and produces a structured metadata payload ready for submission to the CodeDD API.
+
+**No file contents are included** — only paths, types, and line counts.
+"""
+
+import logging
+import os
+import subprocess
+from collections.abc import Callable
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from codedd_cli.scanner.file_classifier import (
+    get_file_type,
+    should_exclude_directory,
+    should_exclude_file,
+)
+from codedd_cli.scanner.line_counter import count_lines
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FileMetadata:
+    """Metadata for a single scanned file (no content)."""
+
+    relative_path: str
+    file_type: str
+    lines_of_code: int
+    lines_of_doc: int
+    selected_for_audit: bool = True
+
+
+@dataclass
+class FolderMetadata:
+    """Aggregated metadata for a folder."""
+
+    relative_path: str
+    lines_of_code: int = 0
+    file_count: int = 0
+
+
+@dataclass
+class ScanResult:
+    """Complete scan result for a single repository directory."""
+
+    root_path: str
+    repo_name: str
+    branch: str
+    commit_hash: str
+    files: list[FileMetadata] = field(default_factory=list)
+    folders: list[FolderMetadata] = field(default_factory=list)
+    total_files: int = 0
+    total_lines_of_code: int = 0
+    total_lines_of_doc: int = 0
+    errors: list[str] = field(default_factory=list)
+
+    def to_dict(self) -> dict:
+        """Serialise to a plain dict suitable for JSON encoding."""
+        return {
+            "root_path": self.root_path,
+            "repo_name": self.repo_name,
+            "branch": self.branch,
+            "commit_hash": self.commit_hash,
+            "total_files": self.total_files,
+            "total_lines_of_code": self.total_lines_of_code,
+            "total_lines_of_doc": self.total_lines_of_doc,
+            "files": [
+                {
+                    "relative_path": f.relative_path,
+                    "file_type": f.file_type,
+                    "lines_of_code": f.lines_of_code,
+                    "lines_of_doc": f.lines_of_doc,
+                    "selected_for_audit": f.selected_for_audit,
+                }
+                for f in self.files
+            ],
+            "folders": [
+                {
+                    "relative_path": fd.relative_path,
+                    "lines_of_code": fd.lines_of_code,
+                    "file_count": fd.file_count,
+                }
+                for fd in self.folders
+            ],
+            "errors": self.errors,
+        }
+
+
+def _git_info(repo_path: str) -> tuple[str, str]:
+    """Return (branch, short_commit_hash) for a repo, or empty strings on error."""
+    try:
+        branch = subprocess.run(
+            ["git", "rev-parse", "--abbrev-ref", "HEAD"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        ).stdout.strip()
+
+        commit = subprocess.run(
+            ["git", "rev-parse", "--short", "HEAD"],
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            timeout=10,
+        ).stdout.strip()
+
+        return branch, commit
+    except Exception:
+        return "", ""
+
+
+def scan_repository(
+    root_path: str,
+    progress_callback: Callable[[int, int, str], None] | None = None,
+) -> ScanResult:
+    """
+    Walk *root_path*, classify every file, and count lines of code.
+
+    Args:
+        root_path:         Absolute path to the repository root.
+        progress_callback: Optional callable(current, total, file_path) invoked
+                           for each file processed.  Useful for Rich progress bars.
+
+    Returns:
+        A ``ScanResult`` containing all file and folder metadata.
+    """
+    root = Path(root_path).resolve()
+    repo_name = root.name
+    branch, commit = _git_info(str(root))
+
+    result = ScanResult(
+        root_path=str(root),
+        repo_name=repo_name,
+        branch=branch,
+        commit_hash=commit,
+    )
+
+    # Phase 1: collect all eligible file paths
+    file_paths: list[str] = []
+    for dirpath, dirnames, filenames in os.walk(str(root)):
+        # Filter out excluded directories in-place so os.walk skips them
+        dirnames[:] = [d for d in dirnames if not should_exclude_directory(d)]
+        for fname in filenames:
+            full_path = os.path.join(dirpath, fname)
+            # Skip symlinks
+            if os.path.islink(full_path):
+                continue
+            file_paths.append(full_path)
+
+    total_files = len(file_paths)
+    folder_aggregates: dict[str, FolderMetadata] = {}
+
+    # Phase 2: classify and count
+    for idx, full_path in enumerate(file_paths):
+        try:
+            rel_path = os.path.relpath(full_path, start=str(root)).replace("\\", "/")
+            file_type = get_file_type(full_path)
+
+            # Count LoC (returns (0, 0) for excluded files)
+            loc, doc = count_lines(full_path)
+
+            fm = FileMetadata(
+                relative_path=rel_path,
+                file_type=file_type,
+                lines_of_code=loc,
+                lines_of_doc=doc,
+                selected_for_audit=(not should_exclude_file(full_path) and loc > 0),
+            )
+            result.files.append(fm)
+
+            if fm.selected_for_audit:
+                result.total_files += 1
+                result.total_lines_of_code += loc
+                result.total_lines_of_doc += doc
+
+            # Aggregate into parent folder and ensure all ancestor folders exist.
+            # The server-side import (import_file_list.py) discovers ALL directories
+            # via os.scandir.  The CLI must produce the same set so the folder
+            # hierarchy can be reconstructed on the server when building TypeDB
+            # directory_content relations (folder → parent_folder → root).
+            parent_rel = os.path.dirname(rel_path).replace("\\", "/")
+            if parent_rel and parent_rel != ".":
+                # Direct parent gets LoC / file_count
+                if parent_rel not in folder_aggregates:
+                    folder_aggregates[parent_rel] = FolderMetadata(relative_path=parent_rel)
+                folder_aggregates[parent_rel].lines_of_code += loc
+                folder_aggregates[parent_rel].file_count += 1
+
+                # Ensure every ancestor folder exists (with 0 direct metrics)
+                ancestor = os.path.dirname(parent_rel).replace("\\", "/")
+                while ancestor and ancestor != ".":
+                    if ancestor not in folder_aggregates:
+                        folder_aggregates[ancestor] = FolderMetadata(relative_path=ancestor)
+                    ancestor = os.path.dirname(ancestor).replace("\\", "/")
+
+            if progress_callback:
+                progress_callback(idx + 1, total_files, rel_path)
+
+        except Exception as exc:
+            error_msg = f"Error scanning {full_path}: {exc}"
+            logger.warning(error_msg)
+            result.errors.append(error_msg)
+
+    # Build sorted folder list
+    result.folders = sorted(folder_aggregates.values(), key=lambda f: f.relative_path)
+
+    return result

codedd_cli/scanner/line_counter.py

@@ -0,0 +1,80 @@
+"""
+Local line-of-code counter.
+
+Counts non-empty lines of code and comment/doc lines for a single file.
+This is a pure-Python implementation that mirrors the server-side
+``get_code_and_doc_lines`` fallback logic — no external ``linecounter``
+binary is required.
+"""
+
+import os
+
+from codedd_cli.scanner.file_classifier import (
+    get_file_type,
+    should_exclude_file,
+)
+
+# File types eligible for LoC counting
+_COUNTABLE_TYPES = frozenset(
+    {
+        "Source Code",
+        "Configuration",
+        "Documentation",
+        "System",
+        "Security",
+        "Other",
+    }
+)
+
+# Single-line comment prefixes by convention
+_COMMENT_PREFIXES = ("#", "//", "/*", "*", "*/", "--", ";", "%", "REM ")
+
+# Maximum file size (in bytes) to attempt reading — skip very large binaries
+_MAX_FILE_SIZE = 50 * 1024 * 1024  # 50 MB
+
+
+def count_lines(file_path: str) -> tuple[int, int]:
+    """
+    Count lines of code and lines of documentation/comments in *file_path*.
+
+    Args:
+        file_path: Absolute path to the file.
+
+    Returns:
+        Tuple of (lines_of_code, lines_of_doc).
+        Returns ``(0, 0)`` for excluded or unreadable files.
+    """
+    if should_exclude_file(file_path):
+        return 0, 0
+
+    file_type = get_file_type(file_path)
+    if file_type not in _COUNTABLE_TYPES:
+        return 0, 0
+
+    # Safety: skip extremely large files
+    try:
+        size = os.path.getsize(file_path)
+        if size > _MAX_FILE_SIZE:
+            return 0, 0
+    except OSError:
+        return 0, 0
+
+    try:
+        with open(file_path, encoding="utf-8", errors="replace") as fh:
+            lines = fh.readlines()
+    except (OSError, PermissionError):
+        return 1, 0  # Minimum fallback
+
+    non_empty = 0
+    comment_lines = 0
+
+    for line in lines:
+        stripped = line.strip()
+        if not stripped:
+            continue
+        non_empty += 1
+        if stripped.startswith(_COMMENT_PREFIXES):
+            comment_lines += 1
+
+    code_lines = max(non_empty - comment_lines, 1) if non_empty > 0 else 0
+    return code_lines, comment_lines

codedd_cli/utils/__init__.py

@@ -0,0 +1 @@
+

codedd_cli/utils/directory_validator.py

@@ -0,0 +1,178 @@
+"""
+Local directory validation for audit scope.
+
+Validates that a user-provided path:
+    1. Exists on the filesystem
+    2. Is a directory (not a file)
+    3. Is readable / accessible by the current user
+    4. Is the root of a Git repository (contains ``.git/``)
+    5. Has at least one commit (valid HEAD)
+
+Uses the **default branch** (main or master) for the audit, not the current
+checkout. Fetches the default branch from origin when a remote exists so
+the scope is tied to the canonical branch state.
+"""
+
+import os
+import subprocess
+from pathlib import Path
+
+from codedd_cli.models.local_directory import LocalDirectory
+
+# Default branch names to try, in order of preference.
+DEFAULT_BRANCH_ORDER = ("main", "master")
+
+
+def validate_directory(raw_path: str) -> LocalDirectory:
+    """
+    Run all validation checks on *raw_path* and return a ``LocalDirectory``.
+
+    The returned object's ``is_valid`` flag indicates whether all checks
+    passed.  When ``is_valid`` is False, the ``error`` field explains why.
+
+    Args:
+        raw_path: A path string as entered by the user.  May be relative;
+                  it is resolved to an absolute path internally.
+
+    Returns:
+        A populated ``LocalDirectory`` instance.
+    """
+    # Normalise and resolve to an absolute path
+    try:
+        resolved = Path(raw_path).expanduser().resolve()
+        abs_path = str(resolved)
+    except (OSError, ValueError) as exc:
+        return LocalDirectory(
+            path=raw_path,
+            is_valid=False,
+            error=f"Invalid path: {exc}",
+        )
+
+    # 1. Existence check
+    if not resolved.exists():
+        return LocalDirectory(
+            path=abs_path,
+            is_valid=False,
+            error="Directory does not exist",
+        )
+
+    # 2. Must be a directory (not a file or symlink to file)
+    if not resolved.is_dir():
+        return LocalDirectory(
+            path=abs_path,
+            is_valid=False,
+            error="Path is not a directory",
+        )
+
+    # 3. Accessibility — try listing the directory contents
+    if not os.access(abs_path, os.R_OK):
+        return LocalDirectory(
+            path=abs_path,
+            is_valid=False,
+            error="Directory is not readable (permission denied)",
+        )
+
+    # 4. Must be a Git repository root
+    git_dir = resolved / ".git"
+    if not git_dir.exists():
+        return LocalDirectory(
+            path=abs_path,
+            repo_name=resolved.name,
+            is_valid=False,
+            error="Not a Git repository (no .git directory found)",
+        )
+
+    # 5. Use default branch (main/master) and its commit for the audit
+    branch, commit_hash, git_error = _default_branch_metadata(abs_path)
+    if git_error:
+        return LocalDirectory(
+            path=abs_path,
+            repo_name=resolved.name,
+            is_valid=False,
+            error=git_error,
+        )
+
+    return LocalDirectory(
+        path=abs_path,
+        repo_name=resolved.name,
+        branch=branch,
+        commit_hash=commit_hash,
+        is_valid=True,
+        error="",
+    )
+
+
+def _run_git(repo_path: str, args: list[str], timeout: int = 15) -> tuple[int, str, str]:
+    """Run a git command; returns (returncode, stdout, stderr)."""
+    try:
+        result = subprocess.run(
+            ["git"] + args,
+            cwd=repo_path,
+            capture_output=True,
+            text=True,
+            timeout=timeout,
+        )
+        return result.returncode, result.stdout.strip(), result.stderr.strip()
+    except FileNotFoundError:
+        return -1, "", "Git is not installed or not in PATH"
+    except subprocess.TimeoutExpired:
+        return -1, "", "Git command timed out"
+    except Exception as exc:
+        return -1, "", str(exc)
+
+
+def _get_default_branch_name(repo_path: str) -> str:
+    """
+    Determine the default branch: origin/HEAD, then local main, then local master.
+
+    Returns the branch name (e.g. "main") or empty string if none found.
+    """
+    # Prefer remote default (e.g. origin/HEAD -> origin/main)
+    code, out, _ = _run_git(repo_path, ["rev-parse", "--abbrev-ref", "origin/HEAD"], timeout=5)
+    if code == 0 and out and out != "origin/HEAD":
+        # "origin/HEAD" -> "origin/main" -> we want "main"
+        if out.startswith("origin/"):
+            return out[7:]
+        return out
+
+    # Fallback: which of main/master exists (local or remote)
+    for branch in DEFAULT_BRANCH_ORDER:
+        code, _, _ = _run_git(repo_path, ["rev-parse", "--verify", branch], timeout=5)
+        if code == 0:
+            return branch
+        code, _, _ = _run_git(repo_path, ["rev-parse", "--verify", f"origin/{branch}"], timeout=5)
+        if code == 0:
+            return branch
+    return ""
+
+
+def _default_branch_metadata(repo_path: str) -> tuple[str, str, str]:
+    """
+    Resolve the default branch (main/master), fetch it from origin if possible,
+    and return its name and short commit hash for use in the audit scope.
+
+    Args:
+        repo_path: Absolute path to the repository root.
+
+    Returns:
+        Tuple of (branch_name, short_commit_hash, error_message).
+        On success ``error_message`` is empty.
+    """
+    branch_name = _get_default_branch_name(repo_path)
+    if not branch_name:
+        # Last resort: use current HEAD so we don't fail local-only repos
+        code, head_branch, err = _run_git(repo_path, ["rev-parse", "--abbrev-ref", "HEAD"])
+        if code != 0:
+            return "", "", f"Git error: no main/master branch and {err or 'unable to read HEAD'}"
+        branch_name = head_branch
+
+    # Fetch the default branch from origin (best-effort; ignore failures for offline/local repos)
+    _run_git(repo_path, ["fetch", "origin", branch_name], timeout=30)
+
+    # Resolve commit: prefer origin/<branch>, then local <branch>
+    for ref in (f"origin/{branch_name}", branch_name):
+        code, commit, err = _run_git(repo_path, ["rev-parse", "--short", ref])
+        if code == 0 and commit:
+            return branch_name, commit, ""
+
+    return branch_name, "", f"Git error: could not resolve commit for branch '{branch_name}'"