source-kb 0.2.2__py3-none-any.whl

core/skeleton/impact.py

@@ -0,0 +1,278 @@
+"""Change impact analysis — maps source file changes to document sections.
+
+Determines which document sections need updating based on changed files
+and their classifications.
+
+Usage:
+    from core.skeleton.impact import analyze_impact, ImpactPlan, SectionImpact
+
+    plan = analyze_impact(
+        changed_files=["src/main/.../UserService.java"],
+        module_dir=knowledge_dir / "point-base",
+        preset=preset,
+        source_cache=cache_dir / "point-base",
+    )
+"""
+
+from __future__ import annotations
+
+import logging
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, Literal
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Data classes
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class SectionImpact:
+    """A single section that needs updating."""
+
+    doc_path: str  # relative filename: "business-logic.md"
+    section_heading: str  # "## Points Issuance Flow"
+    change_type: Literal["update", "create", "delete"]
+    affected_files: list[str]  # source files that triggered this
+    confidence: float = 1.0  # mapping confidence (1.0 = exact match)
+
+
+@dataclass
+class ImpactPlan:
+    """Complete impact analysis result."""
+
+    sections: list[SectionImpact] = field(default_factory=list)
+    unmapped_files: list[str] = field(default_factory=list)
+
+    @property
+    def total_updates(self) -> int:
+        return sum(1 for s in self.sections if s.change_type == "update")
+
+    @property
+    def total_creates(self) -> int:
+        return sum(1 for s in self.sections if s.change_type == "create")
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def analyze_impact(
+    changed_files: list[str],
+    module_dir: Path,
+    preset: dict[str, Any],
+    source_cache: Path,
+    *,
+    change_details: list[dict] | None = None,
+) -> ImpactPlan:
+    """Analyze which document sections are affected by source changes.
+
+    Mapping strategy:
+    1. Classify each changed file using preset file_classification rules
+    2. Determine target document(s) via 'affects' field
+    3. Extract class/method names from file path
+    4. Scan target document headings for matches
+    5. If no heading match, flag as "new content"
+
+    Args:
+        changed_files: List of changed file paths (relative to repo root)
+        module_dir: Module knowledge directory (contains .md docs)
+        preset: Preset configuration dict
+        source_cache: Source code cache directory
+        change_details: Optional git diff details [{file, status, additions, deletions}]
+
+    Returns:
+        ImpactPlan with affected sections and unmapped files
+    """
+    plan = ImpactPlan()
+    classification_rules = preset.get("file_classification", {})
+
+    # Build deletion set for special handling
+    deleted_files: set[str] = set()
+    if change_details:
+        for detail in change_details:
+            if detail.get("status") == "D":
+                deleted_files.add(detail["file"])
+
+    # Group impacts by (doc_path, section_heading) to deduplicate
+    seen: dict[tuple[str, str], SectionImpact] = {}
+
+    for filepath in changed_files:
+        # Classify file → determine target docs
+        target_docs = _classify_to_docs(filepath, classification_rules, preset)
+
+        if not target_docs:
+            plan.unmapped_files.append(filepath)
+            continue
+
+        # Extract search terms from file path/name
+        search_terms = _extract_search_terms(filepath)
+
+        mapped = False
+        for doc_filename in target_docs:
+            doc_path = module_dir / doc_filename
+            if not doc_path.exists():
+                # Doc doesn't exist yet — can't map to sections
+                plan.unmapped_files.append(filepath)
+                mapped = True
+                break
+
+            # Scan document headings for matches
+            matches = _scan_doc_headings_for_match(doc_path, search_terms)
+
+            if matches:
+                for heading, confidence in matches:
+                    change_type: Literal["update", "create", "delete"] = "update"
+                    if filepath in deleted_files:
+                        change_type = "delete"
+
+                    key = (doc_filename, heading)
+                    if key in seen:
+                        seen[key].affected_files.append(filepath)
+                        seen[key].confidence = max(seen[key].confidence, confidence)
+                    else:
+                        impact = SectionImpact(
+                            doc_path=doc_filename,
+                            section_heading=heading,
+                            change_type=change_type,
+                            affected_files=[filepath],
+                            confidence=confidence,
+                        )
+                        seen[key] = impact
+                mapped = True
+            else:
+                # File classified but no section match → new content needed
+                key = (doc_filename, f"## [NEW] {search_terms[0] if search_terms else filepath}")
+                if key not in seen:
+                    impact = SectionImpact(
+                        doc_path=doc_filename,
+                        section_heading=key[1],
+                        change_type="create",
+                        affected_files=[filepath],
+                        confidence=0.5,
+                    )
+                    seen[key] = impact
+                else:
+                    seen[key].affected_files.append(filepath)
+                mapped = True
+
+        if not mapped:
+            plan.unmapped_files.append(filepath)
+
+    plan.sections = list(seen.values())
+    # Sort by confidence descending, then by doc_path
+    plan.sections.sort(key=lambda s: (-s.confidence, s.doc_path, s.section_heading))
+
+    logger.info(
+        "[impact] %d changed files → %d section impacts, %d unmapped",
+        len(changed_files), len(plan.sections), len(plan.unmapped_files),
+    )
+    return plan
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _classify_to_docs(
+    filepath: str,
+    classification_rules: dict[str, Any],
+    preset: dict[str, Any],
+) -> list[str]:
+    """Classify a file and return target document filenames.
+
+    Uses the 'affects' field from file_classification rules.
+    Falls back to pattern matching if no skeleton entry available.
+    """
+    from core.preset_classify import classify_file
+
+    # Classify using full rules (without skeleton entry, pattern-only)
+    categories = classify_file(preset, filepath, skeleton_entry=None)
+
+    # Map categories → affects → doc filenames
+    doc_filenames: list[str] = []
+    for cat in categories:
+        cfg = classification_rules.get(cat, {})
+        affects = cfg.get("affects", [])
+        for doc_file in affects:
+            if doc_file not in doc_filenames:
+                doc_filenames.append(doc_file)
+
+    return doc_filenames
+
+
+def _extract_search_terms(filepath: str) -> list[str]:
+    """Extract search terms from a file path for heading matching.
+
+    Extracts:
+    - Class name (from filename without extension)
+    - Package-derived terms
+    - Endpoint URLs (from common patterns)
+    """
+    terms: list[str] = []
+
+    # Extract filename without extension
+    basename = filepath.rsplit("/", 1)[-1] if "/" in filepath else filepath
+    basename = basename.rsplit("\\", 1)[-1] if "\\" in basename else basename
+    name_no_ext = basename.rsplit(".", 1)[0] if "." in basename else basename
+
+    if name_no_ext:
+        terms.append(name_no_ext)
+
+        # Split CamelCase into words for fuzzy matching
+        words = re.sub(r"([a-z])([A-Z])", r"\1 \2", name_no_ext).split()
+        # Remove common suffixes for better matching
+        clean_words = [w for w in words if w.lower() not in (
+            "service", "impl", "controller", "mapper", "dao",
+            "repository", "config", "configuration", "handler",
+        )]
+        if clean_words and clean_words != [name_no_ext]:
+            terms.append("".join(clean_words))
+
+    return terms
+
+
+def _scan_doc_headings_for_match(
+    doc_path: Path,
+    search_terms: list[str],
+) -> list[tuple[str, float]]:
+    """Scan document headings for terms matching changed entities.
+
+    Returns: [(heading_line, confidence_score)]
+    """
+    try:
+        content = doc_path.read_text(encoding="utf-8", errors="replace")
+    except OSError:
+        return []
+
+    headings: list[str] = []
+    for line in content.splitlines():
+        if line.startswith("## ") or line.startswith("### "):
+            headings.append(line)
+
+    matches: list[tuple[str, float]] = []
+
+    for term in search_terms:
+        term_lower = term.lower()
+        for heading in headings:
+            heading_lower = heading.lower()
+            # Exact class name match in heading
+            if term_lower in heading_lower:
+                # Higher confidence for exact match
+                confidence = 0.95 if term == search_terms[0] else 0.8
+                if (heading, confidence) not in matches:
+                    matches.append((heading, confidence))
+
+    # Deduplicate, keep highest confidence per heading
+    best: dict[str, float] = {}
+    for heading, conf in matches:
+        if heading not in best or conf > best[heading]:
+            best[heading] = conf
+
+    return [(h, c) for h, c in best.items()]

core/skeleton/jar_download.py

@@ -0,0 +1,177 @@
+"""JAR download — HTTP download of Maven source JARs with caching.
+
+Downloads sources.jar files from Maven repositories (Nexus/Artifactory)
+with retry logic, authentication support, and local caching.
+
+On failure: logs warning and returns None (never aborts the pipeline).
+
+Usage:
+    from core.skeleton.jar_download import download_source_jar, is_cached
+
+    if not is_cached(group_id, artifact_id, version, cache_dir):
+        path = download_source_jar(group_id, artifact_id, version, repo_url, cache_dir)
+"""
+
+from __future__ import annotations
+
+import base64
+import logging
+from pathlib import Path
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+logger = logging.getLogger(__name__)
+
+# Constants
+DOWNLOAD_TIMEOUT = 60  # seconds
+MAX_RETRIES = 2
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def download_source_jar(
+    group_id: str,
+    artifact_id: str,
+    version: str,
+    repo_url: str,
+    cache_dir: Path,
+    auth: str | None = None,
+) -> Path | None:
+    """Download a sources.jar from Maven repository.
+
+    Tries sources.jar first, then falls back to regular .jar.
+    Results are cached locally to avoid repeated downloads.
+
+    Args:
+        group_id: Maven groupId (e.g. "com.example")
+        artifact_id: Maven artifactId (e.g. "my-lib")
+        version: Maven version (e.g. "1.2.3")
+        repo_url: Repository base URL
+        cache_dir: Local cache directory
+        auth: Optional "user:password" for Basic auth
+
+    Returns:
+        Path to downloaded JAR file, or None on failure.
+    """
+    # Check cache first
+    cached = _get_cache_path(group_id, artifact_id, version, cache_dir, "sources")
+    if cached.exists() and cached.stat().st_size > 0:
+        logger.debug("Cache hit: %s", cached)
+        return cached
+
+    # Try sources.jar
+    sources_url = _build_maven_url(repo_url, group_id, artifact_id, version, "sources")
+    if _download_file(sources_url, cached, auth):
+        logger.info("Downloaded sources.jar: %s:%s:%s", group_id, artifact_id, version)
+        return cached
+
+    # Try regular .jar as fallback
+    regular_cached = _get_cache_path(group_id, artifact_id, version, cache_dir, "")
+    regular_url = _build_maven_url(repo_url, group_id, artifact_id, version, "")
+    if _download_file(regular_url, regular_cached, auth):
+        logger.info("Downloaded regular jar: %s:%s:%s", group_id, artifact_id, version)
+        return regular_cached
+
+    logger.warning("Failed to download: %s:%s:%s", group_id, artifact_id, version)
+    return None
+
+
+def is_cached(
+    group_id: str,
+    artifact_id: str,
+    version: str,
+    cache_dir: Path,
+) -> bool:
+    """Check if a JAR is already cached locally.
+
+    Args:
+        group_id: Maven groupId
+        artifact_id: Maven artifactId
+        version: Maven version
+        cache_dir: Local cache directory
+
+    Returns:
+        True if sources.jar or regular .jar exists in cache.
+    """
+    sources = _get_cache_path(group_id, artifact_id, version, cache_dir, "sources")
+    if sources.exists() and sources.stat().st_size > 0:
+        return True
+    regular = _get_cache_path(group_id, artifact_id, version, cache_dir, "")
+    return regular.exists() and regular.stat().st_size > 0
+
+
+# ---------------------------------------------------------------------------
+# Internal
+# ---------------------------------------------------------------------------
+
+
+def _get_cache_path(
+    group_id: str, artifact_id: str, version: str,
+    cache_dir: Path, classifier: str,
+) -> Path:
+    """Compute local cache path for a JAR."""
+    group_path = group_id.replace(".", "/")
+    filename = f"{artifact_id}-{version}"
+    if classifier:
+        filename += f"-{classifier}"
+    filename += ".jar"
+    return cache_dir / group_path / artifact_id / version / filename
+
+
+def _build_maven_url(
+    repo_url: str, group_id: str, artifact_id: str,
+    version: str, classifier: str,
+) -> str:
+    """Build Maven repository download URL."""
+    repo_url = repo_url.rstrip("/")
+    group_path = group_id.replace(".", "/")
+    filename = f"{artifact_id}-{version}"
+    if classifier:
+        filename += f"-{classifier}"
+    filename += ".jar"
+    return f"{repo_url}/{group_path}/{artifact_id}/{version}/{filename}"
+
+
+def _download_file(url: str, dest: Path, auth: str | None = None) -> bool:
+    """HTTP download with retry. Returns True on success."""
+    for attempt in range(MAX_RETRIES + 1):
+        try:
+            req = Request(url)
+            if auth:
+                credentials = base64.b64encode(auth.encode()).decode()
+                req.add_header("Authorization", f"Basic {credentials}")
+
+            with urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp:
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                with open(dest, "wb") as f:
+                    while True:
+                        chunk = resp.read(8192)
+                        if not chunk:
+                            break
+                        f.write(chunk)
+            return True
+
+        except HTTPError as e:
+            if e.code == 404:
+                return False
+            if attempt < MAX_RETRIES:
+                logger.debug("HTTP %d on attempt %d: %s", e.code, attempt + 1, url)
+                continue
+            logger.warning("HTTP error %d: %s", e.code, url)
+            return False
+
+        except URLError as e:
+            if attempt < MAX_RETRIES:
+                logger.debug("Network error on attempt %d: %s", attempt + 1, e.reason)
+                continue
+            logger.warning("Network error: %s — %s", e.reason, url)
+            return False
+
+        except OSError as e:
+            logger.warning("IO error downloading %s: %s", url, e)
+            return False
+
+    return False

core/skeleton/jar_resolver.py

@@ -0,0 +1,186 @@
+"""JAR resolver — orchestrate POM parsing, JAR download, and extraction.
+
+Coordinates the full workflow: parse POM → identify internal dependencies →
+download source JARs → extract .java files → return paths.
+
+Uses pom_parser.py for coordinate extraction and jar_download.py for downloading.
+
+Usage:
+    from core.skeleton.jar_resolver import resolve_jar_sources, extract_jar
+
+    paths = resolve_jar_sources(pom_path, cache_dir, repo_url, auth)
+"""
+
+from __future__ import annotations
+
+import logging
+import zipfile
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def resolve_jar_sources(
+    pom_path: Path,
+    cache_dir: Path,
+    repo_url: str,
+    auth: str | None = None,
+    internal_group_ids: list[str] | None = None,
+    known_modules: set[str] | None = None,
+    exclude_artifacts: set[str] | None = None,
+) -> list[Path]:
+    """Orchestrate: parse POM → download JARs → extract → return paths.
+
+    Args:
+        pom_path: Path to pom.xml
+        cache_dir: Directory for caching downloaded JARs and extracted sources
+        repo_url: Maven repository URL
+        auth: Optional "user:password" for Basic auth
+        internal_group_ids: Group ID prefixes to consider internal (auto-inferred if None)
+        known_modules: Module names that already have source (skip these)
+        exclude_artifacts: Artifact IDs to exclude
+
+    Returns:
+        List of directories containing extracted .java source files.
+    """
+    from core.skeleton.pom_parser import parse_pom, extract_dependencies
+    from core.skeleton.jar_download import download_source_jar, is_cached
+
+    if not pom_path.exists():
+        logger.warning("POM file not found: %s", pom_path)
+        return []
+
+    known_modules = known_modules or set()
+    exclude_artifacts = exclude_artifacts or set()
+
+    # Parse POM and get dependencies
+    pom = parse_pom(pom_path)
+    all_deps = extract_dependencies(pom_path)
+
+    # Auto-infer internal group IDs from POM's own groupId
+    if not internal_group_ids:
+        own_gid = pom.get("group_id", "")
+        if own_gid:
+            internal_group_ids = [own_gid]
+        else:
+            logger.warning("Cannot infer internal groupId from POM")
+            return []
+
+    # Filter to internal dependencies
+    internal_deps = _filter_internal_deps(
+        all_deps, internal_group_ids, known_modules, exclude_artifacts
+    )
+
+    if not internal_deps:
+        logger.info("No internal dependencies to resolve")
+        return []
+
+    # Download and extract each dependency
+    result_paths: list[Path] = []
+    for dep in internal_deps:
+        gid = dep["groupId"]
+        aid = dep["artifactId"]
+        version = dep.get("version", "")
+
+        if not version:
+            logger.warning("No version for %s:%s, skipping", gid, aid)
+            continue
+
+        output_dir = cache_dir / f"{aid}-sources"
+
+        # Skip if already extracted
+        if output_dir.exists() and any(output_dir.rglob("*.java")):
+            logger.debug("Already extracted: %s", aid)
+            result_paths.append(output_dir)
+            continue
+
+        # Download JAR
+        jar_path = download_source_jar(gid, aid, version, repo_url, cache_dir, auth)
+        if not jar_path:
+            logger.warning("Failed to download %s:%s:%s", gid, aid, version)
+            continue
+
+        # Extract
+        count = extract_jar(jar_path, output_dir)
+        if count > 0:
+            logger.info("Extracted %d .java files from %s", count, aid)
+            result_paths.append(output_dir)
+        else:
+            logger.warning("No .java files in %s:%s:%s", gid, aid, version)
+
+    return result_paths
+
+
+def extract_jar(jar_path: Path, dest_dir: Path) -> int:
+    """Extract .java files from a JAR (sources.jar or regular .jar).
+
+    Args:
+        jar_path: Path to the JAR file
+        dest_dir: Destination directory for extracted files
+
+    Returns:
+        Number of .java files extracted.
+    """
+    if not jar_path.exists():
+        return 0
+
+    try:
+        count = 0
+        dest_dir.mkdir(parents=True, exist_ok=True)
+
+        with zipfile.ZipFile(jar_path, "r") as zf:
+            for info in zf.infolist():
+                if info.filename.endswith(".java") and not info.is_dir():
+                    zf.extract(info, dest_dir)
+                    count += 1
+
+        return count
+
+    except (zipfile.BadZipFile, OSError) as e:
+        logger.warning("Failed to extract %s: %s", jar_path, e)
+        return 0
+
+
+# ---------------------------------------------------------------------------
+# Internal
+# ---------------------------------------------------------------------------
+
+
+def _filter_internal_deps(
+    all_deps: list[dict],
+    internal_group_ids: list[str],
+    known_modules: set[str],
+    exclude_artifacts: set[str],
+) -> list[dict]:
+    """Filter dependencies to only internal ones that need resolution."""
+    result: list[dict] = []
+
+    for dep in all_deps:
+        gid = dep.get("groupId", "")
+        aid = dep.get("artifactId", "")
+        scope = dep.get("scope", "compile")
+
+        # Skip test/provided scope
+        if scope in ("test", "provided"):
+            continue
+
+        # Check groupId matches internal prefix
+        if not any(gid.startswith(prefix) for prefix in internal_group_ids):
+            continue
+
+        # Skip already-known modules
+        if aid in known_modules:
+            continue
+
+        # Skip excluded artifacts
+        if aid in exclude_artifacts:
+            continue
+
+        result.append(dep)
+
+    return result