graph-dependency-analyzer 0.2.0__py3-none-any.whl

src/infrastructure/parsing/npm_package_dependency_parser.py

@@ -0,0 +1,255 @@
+"""NpmPackageDependencyParser - Parse NPM/Yarn package dependencies.
+
+Extracts dependencies from package.json, package-lock.json, and yarn.lock files.
+"""
+import json
+import logging
+import re
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+class NpmPackageDependencyParser:
+    """
+    NPM/Yarn package dependency parser.
+
+    Extracts dependencies from package.json, package-lock.json, and yarn.lock.
+    """
+
+    def parse_package_json(self, content: str) -> list[dict]:
+        """
+        Parse package.json and extract all dependencies.
+
+        Args:
+            content: Raw package.json file content
+
+        Returns:
+            List of dependency dictionaries (empty list if completely malformed)
+
+        Note:
+            Logs warning and returns empty list if JSON is completely malformed.
+            Logs warning and extracts valid sections if only some sections are malformed.
+        """
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError as e:
+            logger.warning(f"Invalid JSON in package.json: {e}. Returning empty list.")
+            return []
+
+        dependencies = []
+
+        # Extract production dependencies with error handling
+        if "dependencies" in data:
+            try:
+                if isinstance(data["dependencies"], dict):
+                    dependencies.extend(
+                        self._parse_dependency_section(data["dependencies"], "production")
+                    )
+                else:
+                    logger.warning("dependencies section is not an object, skipping")
+            except Exception as e:
+                logger.warning(f"Error parsing dependencies section: {e}")
+
+        # Extract devDependencies with error handling
+        if "devDependencies" in data:
+            try:
+                if isinstance(data["devDependencies"], dict):
+                    dependencies.extend(
+                        self._parse_dependency_section(data["devDependencies"], "development")
+                    )
+                else:
+                    logger.warning("devDependencies section is not an object, skipping")
+            except Exception as e:
+                logger.warning(f"Error parsing devDependencies section: {e}")
+
+        # Extract peerDependencies with error handling
+        if "peerDependencies" in data:
+            try:
+                if isinstance(data["peerDependencies"], dict):
+                    dependencies.extend(
+                        self._parse_dependency_section(data["peerDependencies"], "peer")
+                    )
+                else:
+                    logger.warning("peerDependencies section is not an object, skipping")
+            except Exception as e:
+                logger.warning(f"Error parsing peerDependencies section: {e}")
+
+        return dependencies
+
+    def _parse_dependency_section(
+        self, deps: dict[str, str], scope: str
+    ) -> list[dict]:
+        """Parse a dependency section (dependencies, devDependencies, etc.)."""
+        result = []
+        for name, version in deps.items():
+            group_id, artifact_id = self._parse_package_name(name)
+            result.append(
+                {
+                    "name": name,
+                    "version": version,
+                    "scope": scope,
+                    "group_id": group_id,
+                    "artifact_id": artifact_id,
+                }
+            )
+        return result
+
+    def _parse_package_name(self, name: str) -> tuple[str, str]:
+        """
+        Parse package name into group_id and artifact_id.
+
+        Args:
+            name: Package name (e.g., "@types/react" or "react")
+
+        Returns:
+            Tuple of (group_id, artifact_id)
+        """
+        if name.startswith("@"):
+            # Scoped package: @scope/name
+            parts = name.split("/", 1)
+            if len(parts) == 2:
+                return parts[0], parts[1]
+            return "", name
+        else:
+            # Regular package
+            return "", name
+
+    def parse_package_lock(self, content: str) -> list[dict]:
+        """
+        Parse package-lock.json and extract resolved versions.
+
+        Args:
+            content: Raw package-lock.json file content
+
+        Returns:
+            List of dependency dictionaries with resolved versions and transitive flag
+        """
+        try:
+            data = json.loads(content)
+        except json.JSONDecodeError:
+            return []
+
+        dependencies = []
+        lockfile_version = data.get("lockfileVersion", 1)
+
+        if lockfile_version >= 2:
+            # lockfileVersion 2 and 3 use "packages" object
+            packages = data.get("packages", {})
+
+            # Get direct dependencies from root package
+            root_pkg = packages.get("", {})
+            direct_deps = set()
+            for dep_section in ["dependencies", "devDependencies", "peerDependencies"]:
+                if dep_section in root_pkg:
+                    direct_deps.update(root_pkg[dep_section].keys())
+
+            for pkg_path, pkg_info in packages.items():
+                if pkg_path == "":
+                    # Root package, skip
+                    continue
+
+                # Extract package name from path (e.g., "node_modules/react" -> "react")
+                name = pkg_path.replace("node_modules/", "")
+                version = pkg_info.get("version")
+
+                if name and version:
+                    group_id, artifact_id = self._parse_package_name(name)
+
+                    # Check if this is a transitive dependency
+                    is_transitive = name not in direct_deps
+
+                    dependencies.append(
+                        {
+                            "name": name,
+                            "version": version,
+                            "resolved": True,
+                            "transitive": is_transitive,
+                            "group_id": group_id,
+                            "artifact_id": artifact_id,
+                        }
+                    )
+        else:
+            # lockfileVersion 1 uses "dependencies" object
+            deps = data.get("dependencies", {})
+            dependencies.extend(self._parse_lockfile_v1_dependencies(deps))
+
+        return dependencies
+
+    def _parse_lockfile_v1_dependencies(self, deps: dict) -> list[dict]:
+        """Parse lockfileVersion 1 dependencies recursively."""
+        result = []
+        for name, info in deps.items():
+            version = info.get("version")
+            if version:
+                group_id, artifact_id = self._parse_package_name(name)
+                result.append(
+                    {
+                        "name": name,
+                        "version": version,
+                        "resolved": True,
+                        "group_id": group_id,
+                        "artifact_id": artifact_id,
+                    }
+                )
+
+            # Recursively parse nested dependencies
+            if "dependencies" in info:
+                result.extend(self._parse_lockfile_v1_dependencies(info["dependencies"]))
+
+        return result
+
+    def parse_yarn_lock(self, content: str) -> list[dict]:
+        """
+        Parse yarn.lock and extract resolved versions.
+
+        Args:
+            content: Raw yarn.lock file content
+
+        Returns:
+            List of dependency dictionaries
+
+        Implementation:
+            Uses line-based regex parsing for yarn.lock format
+        """
+        dependencies = []
+        current_package = None
+        current_name = None
+        current_range = None
+
+        lines = content.split("\n")
+        for line in lines:
+            # Package declaration line: react@^18.2.0:
+            if line and not line.startswith(" ") and "@" in line and line.endswith(":"):
+                # Extract package name and version range
+                declaration = line.rstrip(":")
+                # Handle quoted names: "@babel/core@^7.22.0"
+                declaration = declaration.strip('"')
+
+                if "@" in declaration:
+                    # Split by last @ to separate name and version
+                    parts = declaration.rsplit("@", 1)
+                    if len(parts) == 2:
+                        current_name = parts[0]
+                        current_range = parts[1]
+                        current_package = {"name": current_name, "version_range": current_range}
+
+            # Version line:   version "18.2.0"
+            elif line.strip().startswith("version") and current_package:
+                match = re.search(r'version\s+"([^"]+)"', line)
+                if match:
+                    version = match.group(1)
+                    current_package["version"] = version
+
+                    # Parse name into group_id/artifact_id
+                    group_id, artifact_id = self._parse_package_name(current_name)
+                    current_package["group_id"] = group_id
+                    current_package["artifact_id"] = artifact_id
+
+                    # Add to results
+                    dependencies.append(current_package.copy())
+                    current_package = None
+                    current_name = None
+                    current_range = None
+
+        return dependencies

src/infrastructure/parsing/python_parser.py

@@ -0,0 +1,211 @@
+"""
+Python source code parser — extracts classes, functions, imports, and API calls.
+
+Uses regex-based parsing (no AST dependency) to extract:
+  - Module-level imports and from-imports
+  - Class definitions with methods
+  - Function definitions
+  - HTTP calls (requests, httpx, aiohttp, urllib)
+  - Azure SDK usage (azure.identity, azure.appconfiguration, etc.)
+  - URL strings and environment variable references
+  - Database connections (SQLAlchemy, psycopg2, pymongo, etc.)
+
+Returns ParseResult compatible with the existing indexing pipeline.
+"""
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from src.domain.repositories.i_code_parser import ParseResult
+
+
+# ---------------------------------------------------------------------------
+# Regex patterns
+# ---------------------------------------------------------------------------
+
+# import os  /  import sys, json
+_RE_IMPORT = re.compile(r'^import\s+([\w,\s]+)', re.M)
+
+# from azure.identity import DefaultAzureCredential
+_RE_FROM_IMPORT = re.compile(r'^from\s+([\w.]+)\s+import\s+(.+)', re.M)
+
+# class Config:  /  class MyClass(Base):
+_RE_CLASS = re.compile(r'^class\s+(\w+)\s*(?:\(([^)]*)\))?:', re.M)
+
+# def my_function(  /  async def handler(
+_RE_DEF = re.compile(r'^(?:async\s+)?def\s+(\w+)\s*\(', re.M)
+
+# HTTP calls: requests.get("url"), httpx.post("url"), self.client.get("url")
+_RE_HTTP_CALL = re.compile(
+    r'(?:requests|httpx|aiohttp|urllib\.request|self\.\w*(?:client|http|session)\w*)'
+    r'\.(get|post|put|patch|delete|request)\s*\(\s*["\']([^"\']{5,200})["\']',
+    re.I,
+)
+
+# String literals that look like URLs
+_RE_URL_STRING = re.compile(
+    r'["\']((https?://|wss?://)[^\s"\'<>]{5,200})["\']',
+    re.I,
+)
+
+# os.getenv / os.environ references (env var names)
+_RE_ENV_VAR = re.compile(r'os\.(?:getenv|environ\.get)\s*\(\s*["\']([^"\']+)["\']')
+
+# Azure SDK imports
+_RE_AZURE = re.compile(r'(?:from|import)\s+(azure\.[.\w]+)', re.I)
+
+# Database connection strings / SQLAlchemy engine
+_RE_DB_URL = re.compile(
+    r'["\'](?:postgresql|postgres|mysql|sqlite|oracle|mssql|mongodb|redis)'
+    r'(?:\+\w+)?://[^\s"\']{3,200}["\']',
+    re.I,
+)
+
+# Decorators that indicate HTTP endpoints (FastAPI, Flask)
+_RE_ROUTE = re.compile(
+    r'@(?:app|router|blueprint)\.(get|post|put|patch|delete)\s*\(\s*["\']([^"\']+)["\']',
+    re.I,
+)
+
+
+# ---------------------------------------------------------------------------
+# Helper
+# ---------------------------------------------------------------------------
+
+def _line_of(content: str, pos: int) -> int:
+    return content[:pos].count('\n') + 1
+
+
+class PythonParser:
+    """Parser for Python source files."""
+
+    PYTHON_EXTENSIONS = {'.py', '.pyw'}
+
+    def parse_file(self, file_path: Path) -> ParseResult:
+        if not file_path.exists():
+            return ParseResult(file_path=file_path)
+
+        try:
+            content = file_path.read_text(encoding='utf-8', errors='replace')
+        except Exception:
+            return ParseResult(file_path=file_path)
+
+        if file_path.suffix.lower() not in self.PYTHON_EXTENSIONS:
+            return ParseResult(file_path=file_path)
+
+        # ── Derive module qualified name from path ─────────────────────────
+        # e.g. shared/config.py → shared.config
+        stem_parts = list(file_path.parts)
+        # Find 'src' or repo root anchor
+        module_name = file_path.stem
+        try:
+            for anchor in ('src', 'app', 'lib'):
+                if anchor in stem_parts:
+                    idx = stem_parts.index(anchor)
+                    parts = stem_parts[idx:]
+                    module_name = '.'.join(
+                        p for p in parts[:-1]  # dirs
+                    ) + '.' + file_path.stem
+                    break
+        except Exception:
+            pass
+
+        # ── Classes ──────────────────────────────────────────────────────
+        classes = []
+        for m in _RE_CLASS.finditer(content):
+            class_name = m.group(1)
+            bases = m.group(2) or ''
+            qname = f"{module_name}.{class_name}"
+            classes.append({
+                'name': class_name,
+                'qualified_name': qname,
+                'node_type': 'CLASS',
+                'line': _line_of(content, m.start()),
+                'metadata': {
+                    'language': 'python',
+                    'base_classes': bases.strip(),
+                    'file_path': str(file_path),
+                }
+            })
+
+        # ── Functions / methods ───────────────────────────────────────────
+        methods = []
+        for m in _RE_DEF.finditer(content):
+            fn_name = m.group(1)
+            if fn_name.startswith('_') and fn_name != '__init__':
+                continue  # skip private helpers
+            qname = f"{module_name}.{fn_name}"
+            methods.append({
+                'name': fn_name,
+                'qualified_name': qname,
+                'node_type': 'METHOD',
+                'line': _line_of(content, m.start()),
+                'metadata': {
+                    'language': 'python',
+                    'file_path': str(file_path),
+                }
+            })
+
+        # ── Imports ───────────────────────────────────────────────────────
+        imports = []
+        seen_imports: set[str] = set()
+
+        for m in _RE_IMPORT.finditer(content):
+            for mod in m.group(1).split(','):
+                mod = mod.strip().split('\n')[0].strip()  # take only first line
+                if mod and '.' not in mod or mod.count('.') < 3:  # skip very long chains
+                    if mod and mod not in seen_imports:
+                        seen_imports.add(mod)
+                        imports.append({
+                            'import_path': mod,
+                            'line': _line_of(content, m.start()),
+                        })
+
+        for m in _RE_FROM_IMPORT.finditer(content):
+            mod = m.group(1).strip()
+            if mod and mod not in seen_imports:
+                seen_imports.add(mod)
+                imports.append({
+                    'import_path': mod,
+                    'line': _line_of(content, m.start()),
+                })
+
+        # ── Method calls / API refs ───────────────────────────────────────
+        method_calls = []
+
+        # HTTP calls with URL
+        for m in _RE_HTTP_CALL.finditer(content):
+            method_calls.append({
+                'callee': m.group(0).split('(')[0].strip(),
+                'is_api_call': True,
+                'api_path': m.group(2),
+                'line': _line_of(content, m.start()),
+            })
+
+        # FastAPI / Flask route decorators → ENDPOINT
+        for m in _RE_ROUTE.finditer(content):
+            method_calls.append({
+                '_is_endpoint': True,
+                'http_method': m.group(1).upper(),
+                'path': m.group(2),
+                'line': _line_of(content, m.start()),
+            })
+
+        # ── String literals (URLs + env vars) ─────────────────────────────
+        string_literals: list[str] = []
+        for m in _RE_URL_STRING.finditer(content):
+            string_literals.append(m.group(1))
+        for m in _RE_ENV_VAR.finditer(content):
+            string_literals.append(f"env:{m.group(1)}")
+        for m in _RE_DB_URL.finditer(content):
+            string_literals.append(m.group(0).strip('"\''))
+
+        return ParseResult(
+            file_path=file_path,
+            classes=classes,
+            methods=methods,
+            imports=imports,
+            method_calls=method_calls,
+            string_literals=string_literals,
+        )