devflow-toolkit 0.3.0__py3-none-any.whl

devflow/core/project.py

@@ -0,0 +1,533 @@
+"""Project inspection and technology detection."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import tomllib
+from pathlib import Path
+from typing import Any, Iterator
+
+from devflow.models.project import Project
+
+SKIP_DIRECTORIES = {
+  ".git",
+  ".venv",
+  "venv",
+  "env",
+  "node_modules",
+  "__pycache__",
+  ".mypy_cache",
+  ".pytest_cache",
+  ".ruff_cache",
+  ".tox",
+  ".nox",
+  "dist",
+  "build",
+  "coverage",
+  "htmlcov",
+}
+
+SOURCE_EXTENSIONS = {
+  ".py",
+  ".js",
+  ".jsx",
+  ".ts",
+  ".tsx",
+  ".vue",
+  ".svelte",
+  ".go",
+  ".rs",
+  ".php",
+  ".rb",
+  ".java",
+  ".kt",
+}
+
+FRAMEWORK_DEPENDENCIES = {
+  "django": "Django",
+  "djangorestframework": "Django REST Framework",
+  "fastapi": "FastAPI",
+  "flask": "Flask",
+  "starlette": "Starlette",
+  "express": "Express",
+  "nestjs": "NestJS",
+  "react": "React",
+  "vue": "Vue",
+  "svelte": "Svelte",
+  "next": "Next.js",
+  "langchain": "LangChain",
+  "llama-index": "LlamaIndex",
+  "chromadb": "ChromaDB",
+  "faiss-cpu": "FAISS",
+  "sentence-transformers": "Sentence Transformers",
+  "openai": "OpenAI",
+  "anthropic": "Anthropic",
+}
+
+AI_DEPENDENCIES = {
+  "langchain",
+  "llama-index",
+  "chromadb",
+  "faiss-cpu",
+  "sentence-transformers",
+  "pgvector",
+  "qdrant-client",
+  "weaviate-client",
+  "pinecone-client",
+}
+
+FRONTEND_FRAMEWORKS = {
+  "React",
+  "Vue",
+  "Svelte",
+  "Next.js",
+}
+
+BACKEND_FRAMEWORKS = {
+  "Django",
+  "Django REST Framework",
+  "FastAPI",
+  "Flask",
+  "Starlette",
+  "Express",
+  "NestJS",
+}
+
+DATABASE_DEPENDENCIES = {
+  "postgresql": {
+    "psycopg",
+    "psycopg2",
+    "asyncpg",
+    "pg8000",
+  },
+  "mysql": {
+    "mysqlclient",
+    "pymysql",
+    "aiomysql",
+  },
+  "mongodb": {
+    "pymongo",
+    "motor",
+    "beanie",
+  },
+  "sqlite": {
+    "aiosqlite",
+  },
+}
+
+_PROJECT_MARKERS = (
+  ".devflow/config.yaml",
+  "pyproject.toml",
+  "manage.py",
+  "package.json",
+  ".git",
+  "go.mod",
+  "Cargo.toml",
+)
+
+
+def canonicalize_dependency(name: str) -> str:
+  """Normalize a package name for comparisons."""
+  return re.sub(r"[-_.]+", "-", name.strip().lower())
+
+
+def _dependency_name(requirement: str) -> str | None:
+  """Extract a package name from a requirements.txt entry."""
+  value = requirement.strip()
+  if not value or value.startswith("#"):
+    return None
+  if value.startswith("-"):
+    return None
+
+  value = value.split(";", maxsplit=1)[0].strip()
+  if not value:
+    return None
+
+  if " @ " in value:
+    value = value.split(" @ ", maxsplit=1)[0].strip()
+
+  match = re.match(r"([A-Za-z0-9][A-Za-z0-9._-]*)", value)
+  if not match:
+    return None
+  return canonicalize_dependency(match.group(1))
+
+
+def _is_symlink_escape(path: Path, root: Path) -> bool:
+  """Return whether a path is a symlink or resolves outside the root."""
+  if path.is_symlink():
+    return True
+  try:
+    path.resolve().relative_to(root.resolve())
+  except ValueError:
+    return True
+  return False
+
+
+def iter_project_files(
+  root: Path,
+  extensions: set[str] | None = None,
+  max_files: int = 5000,
+) -> Iterator[Path]:
+  """Yield project files while skipping generated and vendor directories."""
+  resolved_root = root.resolve()
+  count = 0
+  for dirpath, dirnames, filenames in os.walk(resolved_root):
+    dirnames[:] = sorted(
+      name
+      for name in dirnames
+      if name not in SKIP_DIRECTORIES
+      and not name.startswith(".")
+      and not (Path(dirpath) / name).is_symlink()
+    )
+
+    for filename in sorted(filenames):
+      if filename.startswith("."):
+        continue
+      path = Path(dirpath) / filename
+      if _is_symlink_escape(path, resolved_root):
+        continue
+      if extensions is not None and path.suffix not in extensions:
+        continue
+      yield path
+      count += 1
+      if count >= max_files:
+        return
+
+
+class ProjectDetector:
+  """Inspect a project and detect its stack and applicable profiles."""
+
+  def find_root(self, start: Path) -> Path:
+    """Find the nearest likely project root from a starting path."""
+    current = start.resolve()
+    if current.is_file():
+      current = current.parent
+
+    for candidate in (current, *current.parents):
+      if any(
+        (candidate / marker).exists()
+        for marker in _PROJECT_MARKERS
+      ):
+        return candidate
+    return current
+
+  def detect(self, root: Path) -> Project:
+    """Inspect a project and return its detected context."""
+    resolved = root.resolve()
+    if not resolved.exists():
+      msg = f"Project path does not exist: {resolved}"
+      raise FileNotFoundError(msg)
+    if not resolved.is_dir():
+      msg = f"Project path is not a directory: {resolved}"
+      raise NotADirectoryError(msg)
+
+    pyproject, pyproject_name = self._read_pyproject(resolved)
+    package_json = self._read_package_json(resolved)
+
+    dependencies: set[str] = set()
+    manifests: list[str] = []
+
+    if pyproject is not None:
+      manifests.append("pyproject.toml")
+      dependencies.update(self._pyproject_dependencies(pyproject))
+
+    if package_json is not None:
+      manifests.append("package.json")
+      dependencies.update(self._package_dependencies(package_json))
+
+    requirements = self._requirements_files(resolved)
+    for path in requirements:
+      manifests.append(path.relative_to(resolved).as_posix())
+      dependencies.update(self._read_requirements(path))
+
+    source_files = [
+      path.relative_to(resolved).as_posix()
+      for path in iter_project_files(
+        resolved,
+        extensions=SOURCE_EXTENSIONS,
+      )
+    ]
+
+    languages = self._detect_languages(
+      resolved,
+      dependencies,
+      source_files,
+    )
+    frameworks = self._detect_frameworks(dependencies)
+    databases = self._detect_databases(dependencies)
+    project_type = self._detect_project_type(frameworks)
+
+    name = pyproject_name
+    if not name and package_json is not None:
+      value = package_json.get("name")
+      if isinstance(value, str) and value.strip():
+        name = value.strip()
+    if not name:
+      name = resolved.name
+
+    return Project(
+      root=resolved,
+      name=name,
+      project_type=project_type,
+      languages=languages,
+      frameworks=frameworks,
+      databases=databases,
+      dependencies=dependencies,
+      manifests=manifests,
+      source_files=source_files,
+    )
+
+  def _project_file(self, root: Path, name: str) -> Path | None:
+    """Return a contained project file, ignoring symlinks and escapes."""
+    path = root / name
+    if not path.is_file() or _is_symlink_escape(path, root):
+      return None
+    return path
+
+  def _read_pyproject(
+    self,
+    root: Path,
+  ) -> tuple[dict[str, Any] | None, str | None]:
+    path = self._project_file(root, "pyproject.toml")
+    if path is None:
+      return None, None
+
+    try:
+      with path.open("rb") as handle:
+        data = tomllib.load(handle)
+    except (OSError, tomllib.TOMLDecodeError):
+      return None, None
+
+    project = data.get("project")
+    name = None
+    if isinstance(project, dict):
+      value = project.get("name")
+      if isinstance(value, str) and value.strip():
+        name = value.strip()
+
+    return data, name
+
+  def _read_package_json(
+    self,
+    root: Path,
+  ) -> dict[str, Any] | None:
+    path = self._project_file(root, "package.json")
+    if path is None:
+      return None
+
+    try:
+      with path.open(encoding="utf-8") as handle:
+        data = json.load(handle)
+    except (OSError, json.JSONDecodeError):
+      return None
+    return data if isinstance(data, dict) else None
+
+  def _pyproject_dependencies(
+    self,
+    data: dict[str, Any],
+  ) -> set[str]:
+    dependencies: set[str] = set()
+    project = data.get("project")
+    if isinstance(project, dict):
+      dependencies.update(
+        self._parse_requirements(project.get("dependencies"))
+      )
+
+      optional = project.get("optional-dependencies")
+      if isinstance(optional, dict):
+        for values in optional.values():
+          dependencies.update(self._parse_requirements(values))
+
+    groups = data.get("dependency-groups")
+    if isinstance(groups, dict):
+      for values in groups.values():
+        dependencies.update(self._parse_requirements(values))
+
+    tool = data.get("tool")
+    poetry = tool.get("poetry") if isinstance(tool, dict) else None
+    if isinstance(poetry, dict):
+      poetry_dependencies = poetry.get("dependencies")
+      if isinstance(poetry_dependencies, dict):
+        for name in poetry_dependencies:
+          if name.lower() != "python":
+            dependencies.add(canonicalize_dependency(name))
+
+      poetry_groups = poetry.get("group")
+      if isinstance(poetry_groups, dict):
+        for group in poetry_groups.values():
+          if not isinstance(group, dict):
+            continue
+          values = group.get("dependencies")
+          if isinstance(values, dict):
+            for name in values:
+              dependencies.add(canonicalize_dependency(name))
+
+    return dependencies
+
+  def _package_dependencies(
+    self,
+    data: dict[str, Any],
+  ) -> set[str]:
+    dependencies: set[str] = set()
+    for section in (
+      "dependencies",
+      "devDependencies",
+      "peerDependencies",
+      "optionalDependencies",
+    ):
+      values = data.get(section)
+      if isinstance(values, dict):
+        for name in values:
+          dependencies.add(canonicalize_dependency(name))
+    return dependencies
+
+  def _requirements_files(self, root: Path) -> list[Path]:
+    candidates = list(root.glob("requirements*.txt"))
+    requirements_dir = root / "requirements"
+    if requirements_dir.is_dir() and not requirements_dir.is_symlink():
+      candidates.extend(requirements_dir.glob("*.txt"))
+
+    paths = {
+      path
+      for path in candidates
+      if path.is_file() and not _is_symlink_escape(path, root)
+    }
+    return sorted(paths)
+
+  def _read_requirements(self, path: Path) -> set[str]:
+    try:
+      lines = path.read_text(encoding="utf-8").splitlines()
+    except OSError:
+      return set()
+    return self._parse_requirements(lines)
+
+  def _parse_requirements(self, values: Any) -> set[str]:
+    if not isinstance(values, list):
+      return set()
+    dependencies = set()
+    for value in values:
+      if not isinstance(value, str):
+        continue
+      name = _dependency_name(value)
+      if name:
+        dependencies.add(name)
+    return dependencies
+
+  def _detect_languages(
+    self,
+    root: Path,
+    dependencies: set[str],
+    source_files: list[str],
+  ) -> list[str]:
+    languages: list[str] = []
+    suffixes = {Path(name).suffix for name in source_files}
+
+    has_python = (
+      ".py" in suffixes
+      or "python" in dependencies
+      or any(
+        self._project_file(root, marker) is not None
+        for marker in ("manage.py", "requirements.txt")
+      )
+    )
+    if has_python:
+      languages.append("python")
+
+    has_package_json = self._project_file(root, "package.json") is not None
+    has_typescript = (
+      ".ts" in suffixes
+      or ".tsx" in suffixes
+      or "typescript" in dependencies
+    )
+    if has_package_json or ".js" in suffixes or ".jsx" in suffixes:
+      languages.append("javascript")
+    if has_typescript:
+      languages.append("typescript")
+
+    if self._project_file(root, "go.mod") is not None or ".go" in suffixes:
+      languages.append("go")
+    if self._project_file(root, "Cargo.toml") is not None or ".rs" in suffixes:
+      languages.append("rust")
+    if (
+      self._project_file(root, "composer.json") is not None
+      or ".php" in suffixes
+    ):
+      languages.append("php")
+    if self._project_file(root, "Gemfile") is not None or ".rb" in suffixes:
+      languages.append("ruby")
+    if ".java" in suffixes:
+      languages.append("java")
+
+    return languages
+
+  def _detect_frameworks(
+    self,
+    dependencies: set[str],
+  ) -> list[str]:
+    frameworks: list[str] = []
+    for dependency, label in FRAMEWORK_DEPENDENCIES.items():
+      if self._has_dependency(dependencies, dependency):
+        frameworks.append(label)
+    return frameworks
+
+  def _detect_databases(
+    self,
+    dependencies: set[str],
+  ) -> list[str]:
+    databases: list[str] = []
+    for label, names in DATABASE_DEPENDENCIES.items():
+      if any(
+        self._has_dependency(dependencies, name)
+        for name in names
+      ):
+        databases.append(label)
+
+    if "chromadb" in dependencies:
+      if "vector-database" not in databases:
+        databases.append("vector-database")
+    return databases
+
+  def _detect_project_type(self, frameworks: list[str]) -> str:
+    frontend = any(
+      framework in FRONTEND_FRAMEWORKS
+      for framework in frameworks
+    )
+    backend = any(
+      framework in BACKEND_FRAMEWORKS
+      for framework in frameworks
+    )
+    ai = any(
+      framework
+      in {
+        "LangChain",
+        "LlamaIndex",
+        "ChromaDB",
+        "FAISS",
+        "Sentence Transformers",
+      }
+      for framework in frameworks
+    )
+
+    if ai:
+      return "ai"
+    if frontend and backend:
+      return "fullstack"
+    if backend:
+      return "backend"
+    if frontend:
+      return "frontend"
+    return "unknown"
+
+  def _has_dependency(
+    self,
+    dependencies: set[str],
+    name: str,
+  ) -> bool:
+    canonical = canonicalize_dependency(name)
+    return any(
+      dependency == canonical
+      or dependency.startswith(f"{canonical}-")
+      for dependency in dependencies
+    )