pi-python-helper 0.1.0 → 0.2.0

package/src/build/traceback.ts

@@ -0,0 +1,55 @@
+export interface TracebackFrame {
+  path: string;
+  line: number;
+  func: string;
+  /** True for site-packages, the standard library, and pytest internals. */
+  library: boolean;
+}
+
+const FRAME_RE = /^\s*File "([^"]+)", line (\d+), in (.+?)\s*$/;
+/**
+ * pytest `--tb=short` replaces the `File "..."` form with `path:line: in func`,
+ * so both notations must be recognised or short tracebacks yield no frame at all.
+ */
+const PYTEST_FRAME_RE = /^\s*([^\s:]+\.py):(\d+): in (.+?)\s*$/;
+
+/**
+ * A frame belongs to library code when it sits in an installed distribution or
+ * in the interpreter's own library tree. Pointing the agent at those frames is
+ * how it ends up editing site-packages instead of the project.
+ */
+export function isLibraryFrame(path: string): boolean {
+  const normalized = path.replace(/\\/g, '/');
+  return (
+    /\/site-packages\//.test(normalized) ||
+    /\/dist-packages\//.test(normalized) ||
+    /\/lib\/python3\.\d+\//.test(normalized) ||
+    /\/python3\.\d+\//.test(normalized) ||
+    /<frozen /.test(normalized) ||
+    /\/_pytest\//.test(normalized) ||
+    /\/pluggy\//.test(normalized)
+  );
+}
+
+export function extractTracebackFrames(output: string): TracebackFrame[] {
+  const frames: TracebackFrame[] = [];
+  for (const line of output.split(/\r?\n/)) {
+    const match = line.match(FRAME_RE) ?? line.match(PYTEST_FRAME_RE);
+    if (!match) continue;
+    const path = match[1];
+    frames.push({
+      path,
+      line: Number.parseInt(match[2], 10),
+      func: match[3].trim(),
+      library: isLibraryFrame(path),
+    });
+  }
+  return frames;
+}
+
+export function firstUserFrame(frames: TracebackFrame[]): TracebackFrame | undefined {
+  for (let index = frames.length - 1; index >= 0; index -= 1) {
+    if (!frames[index].library) return frames[index];
+  }
+  return undefined;
+}

package/src/dependencies/aliases.ts

@@ -0,0 +1,101 @@
+/**
+ * Import name and distribution name frequently disagree. The scanner supplies
+ * the authoritative mapping when the analysing interpreter has the package
+ * installed; this table covers the cases where it does not, so an uninstalled
+ * checkout is still analysed correctly.
+ */
+export const IMPORT_ALIASES: Record<string, string[]> = {
+  PIL: ['pillow'],
+  yaml: ['pyyaml'],
+  dateutil: ['python-dateutil'],
+  bs4: ['beautifulsoup4'],
+  cv2: ['opencv-python', 'opencv-python-headless'],
+  sklearn: ['scikit-learn'],
+  skimage: ['scikit-image'],
+  dotenv: ['python-dotenv'],
+  attr: ['attrs'],
+  attrs: ['attrs'],
+  jwt: ['pyjwt'],
+  jose: ['python-jose'],
+  serial: ['pyserial'],
+  OpenSSL: ['pyopenssl'],
+  Crypto: ['pycryptodome'],
+  pkg_resources: ['setuptools'],
+  MySQLdb: ['mysqlclient'],
+  googleapiclient: ['google-api-python-client'],
+  github: ['PyGithub'],
+  pytest_cov: ['pytest-cov'],
+  _pytest: ['pytest'],
+  pytest_asyncio: ['pytest-asyncio'],
+  psycopg: ['psycopg', 'psycopg-binary'],
+  psycopg2: ['psycopg2', 'psycopg2-binary'],
+  prometheus_client: ['prometheus-client'],
+  grpc: ['grpcio'],
+  kafka: ['kafka-python'],
+  docker: ['docker'],
+  numpy: ['numpy'],
+  pandas: ['pandas'],
+  ruamel: ['ruamel.yaml'],
+  setuptools: ['setuptools'],
+  mako: ['Mako'],
+  pytz: ['pytz'],
+  tzlocal: ['tzlocal'],
+  win32com: ['pywin32'],
+  lxml: ['lxml'],
+  matplotlib: ['matplotlib'],
+  seaborn: ['seaborn'],
+  sqlalchemy: ['sqlalchemy', 'SQLAlchemy'],
+  pydantic: ['pydantic'],
+  fastapi: ['fastapi'],
+  starlette: ['starlette'],
+  uvicorn: ['uvicorn'],
+  celery: ['celery'],
+  redis: ['redis'],
+  boto3: ['boto3'],
+  botocore: ['botocore'],
+  httpx: ['httpx'],
+  aiohttp: ['aiohttp'],
+  werkzeug: ['werkzeug'],
+  flask: ['flask'],
+  django: ['django'],
+  jinja2: ['jinja2'],
+  typer: ['typer'],
+  click: ['click'],
+  rich: ['rich'],
+  tqdm: ['tqdm'],
+};
+
+/** Distributions that are normally invoked as a console script, not imported. */
+export const CONSOLE_ONLY: Set<string> = new Set([
+  'ruff',
+  'mypy',
+  'pyright',
+  'ty',
+  'pytest',
+  'pytest-cov',
+  'coverage',
+  'pre-commit',
+  'tox',
+  'nox',
+  'hatch',
+  'hatchling',
+  'build',
+  'twine',
+  'black',
+  'isort',
+  'flake8',
+  'pylint',
+  'sphinx',
+  'mkdocs',
+  'uvicorn',
+  'gunicorn',
+  'alembic',
+  'celery',
+  'honcho',
+  'maturin',
+  'setuptools-scm',
+  'pip-audit',
+  'bandit',
+  'commitizen',
+  'towncrier',
+]);

package/src/dependencies/plan.ts

@@ -3,109 +3,8 @@ import { warn } from '../core/result.ts';
 import { isTestFile } from '../project/paths.ts';
 import type { DeclaredDependency, ScanPayload } from '../project/scanner.ts';
 
-/**
- * Import name and distribution name frequently disagree. The scanner supplies
- * the authoritative mapping when the analysing interpreter has the package
- * installed; this table covers the cases where it does not, so an uninstalled
- * checkout is still analysed correctly.
- */
-export const IMPORT_ALIASES: Record<string, string[]> = {
-  PIL: ['pillow'],
-  yaml: ['pyyaml'],
-  dateutil: ['python-dateutil'],
-  bs4: ['beautifulsoup4'],
-  cv2: ['opencv-python', 'opencv-python-headless'],
-  sklearn: ['scikit-learn'],
-  skimage: ['scikit-image'],
-  dotenv: ['python-dotenv'],
-  attr: ['attrs'],
-  attrs: ['attrs'],
-  jwt: ['pyjwt'],
-  jose: ['python-jose'],
-  serial: ['pyserial'],
-  OpenSSL: ['pyopenssl'],
-  Crypto: ['pycryptodome'],
-  pkg_resources: ['setuptools'],
-  MySQLdb: ['mysqlclient'],
-  googleapiclient: ['google-api-python-client'],
-  github: ['PyGithub'],
-  pytest_cov: ['pytest-cov'],
-  _pytest: ['pytest'],
-  pytest_asyncio: ['pytest-asyncio'],
-  psycopg: ['psycopg', 'psycopg-binary'],
-  psycopg2: ['psycopg2', 'psycopg2-binary'],
-  prometheus_client: ['prometheus-client'],
-  grpc: ['grpcio'],
-  kafka: ['kafka-python'],
-  docker: ['docker'],
-  numpy: ['numpy'],
-  pandas: ['pandas'],
-  ruamel: ['ruamel.yaml'],
-  setuptools: ['setuptools'],
-  mako: ['Mako'],
-  pytz: ['pytz'],
-  tzlocal: ['tzlocal'],
-  win32com: ['pywin32'],
-  lxml: ['lxml'],
-  matplotlib: ['matplotlib'],
-  seaborn: ['seaborn'],
-  sqlalchemy: ['sqlalchemy', 'SQLAlchemy'],
-  pydantic: ['pydantic'],
-  fastapi: ['fastapi'],
-  starlette: ['starlette'],
-  uvicorn: ['uvicorn'],
-  celery: ['celery'],
-  redis: ['redis'],
-  boto3: ['boto3'],
-  botocore: ['botocore'],
-  httpx: ['httpx'],
-  aiohttp: ['aiohttp'],
-  werkzeug: ['werkzeug'],
-  flask: ['flask'],
-  django: ['django'],
-  jinja2: ['jinja2'],
-  typer: ['typer'],
-  click: ['click'],
-  rich: ['rich'],
-  tqdm: ['tqdm'],
-};
-
-/** Distributions that are normally invoked as a console script, not imported. */
-const CONSOLE_ONLY = new Set(
-  [
-    'ruff',
-    'mypy',
-    'pyright',
-    'ty',
-    'pytest',
-    'pytest-cov',
-    'coverage',
-    'pre-commit',
-    'tox',
-    'nox',
-    'hatch',
-    'hatchling',
-    'build',
-    'twine',
-    'black',
-    'isort',
-    'flake8',
-    'pylint',
-    'sphinx',
-    'mkdocs',
-    'uvicorn',
-    'gunicorn',
-    'alembic',
-    'celery',
-    'honcho',
-    'maturin',
-    'setuptools-scm',
-    'pip-audit',
-    'bandit',
-    'commitizen',
-    'towncrier',
-  ].map((name) => name),
-);
+import { CONSOLE_ONLY, IMPORT_ALIASES } from './aliases.ts';
+export { CONSOLE_ONLY, IMPORT_ALIASES };
 
 const NORMALIZE_RE = /[-_.]+/g;
 
@@ -164,7 +63,16 @@ export interface UndeclaredImport {
   files: string[];
   fileCount: number;
   providers: string[];
-  suggestedDistribution: string;
+  /**
+   * The distribution to declare, when the analysing interpreter could determine
+   * it. Absent otherwise: `uv add <import name>` would then install a different
+   * package, or nothing at all, because import names and distribution names
+   * frequently disagree (`wconfig` ships inside `wpyconf`, `yaml` inside
+   * `PyYAML`).
+   */
+  suggestedDistribution?: string;
+  /** True when the suggestion comes from installed metadata rather than a guess. */
+  providerKnown: boolean;
   typeCheckingOnly: boolean;
   reason: string;
 }
@@ -196,6 +104,12 @@ export interface DependencyPlan {
     requiresPythonMismatch: { manifest: string; lock: string } | null;
   };
   providerMappingReliable: boolean;
+  /**
+   * Third-party imports the analysing interpreter could not map to an installed
+   * distribution. A high count means the interpreter is not the project's own,
+   * so "undeclared" may really be "import name differs from distribution name".
+   */
+  unmappedImports: number;
   unparsable: { path: string; error: string }[];
   warnings: Diagnostic[];
   notes: Diagnostic[];
@@ -248,13 +162,19 @@ export function planDependencies(
     const runtimeRelevant = !entry.typeCheckingOnly && runtimeFiles.length > 0;
 
     if (matched.length === 0) {
+      // Prefer the distribution that actually owns the module. The static alias
+      // table is a fallback for checkouts where nothing is installed.
+      const suggested =
+        entry.providers[0] ??
+        IMPORT_ALIASES[entry.import]?.[0] ??
+        IMPORT_ALIASES[normalizeName(entry.import)]?.[0];
       undeclared.push({
         import: entry.import,
         files: entry.files,
         fileCount: entry.fileCount,
         providers: entry.providers,
-        suggestedDistribution:
-          entry.providers[0] ?? IMPORT_ALIASES[entry.import]?.[0] ?? entry.import,
+        suggestedDistribution: suggested,
+        providerKnown: suggested !== undefined,
         typeCheckingOnly: entry.typeCheckingOnly,
         reason: entry.typeCheckingOnly
           ? 'imported only under TYPE_CHECKING and declared in neither [project] tables nor uv.lock'
@@ -300,11 +220,21 @@ export function planDependencies(
         entry.files[0],
       ),
     );
-    suggestions.push({
-      message: `Declare ${entry.suggestedDistribution} with uv add${entry.typeCheckingOnly ? ' --dev' : ''} ${entry.suggestedDistribution}.`,
-      confidence: entry.providers.length ? 'high' : 'medium',
-      command: `uv add${entry.typeCheckingOnly ? ' --dev' : ''} ${entry.suggestedDistribution}`,
-    });
+    const distribution = entry.suggestedDistribution;
+    if (distribution) {
+      const flag = entry.typeCheckingOnly ? ' --dev' : '';
+      suggestions.push({
+        message: `Declare ${distribution} with uv add${flag} ${distribution}.`,
+        confidence: entry.providers.length ? 'high' : 'medium',
+        command: `uv add${flag} ${distribution}`,
+      });
+    } else {
+      // Fabricating a command here would install the wrong package or fail.
+      suggestions.push({
+        message: `Look up the distribution that provides "${entry.import}" and declare that name: import names and distribution names frequently disagree, and the analysing interpreter could not map this one.`,
+        confidence: 'low',
+      });
+    }
   }
 
   for (const entry of misplaced) {
@@ -353,6 +283,11 @@ export function planDependencies(
     });
   }
 
+  const thirdPartyCount = imports?.thirdParty.length ?? 0;
+  const unmappedImports = (imports?.thirdParty ?? []).filter(
+    (entry) => entry.providers.length === 0,
+  ).length;
+
   if (imports?.providersUnavailable) {
     notes.push({
       code: 'PROVIDER_MAPPING_HEURISTIC',
@@ -360,6 +295,12 @@ export function planDependencies(
         'No installed distributions were visible to the analysing interpreter, so import-to-distribution mapping relied on a static alias table.',
       severity: 'info',
     });
+  } else if (unmappedImports > 0) {
+    notes.push({
+      code: 'UNMAPPED_IMPORTS',
+      message: `${unmappedImports} of ${thirdPartyCount} third-party import(s) could not be mapped to an installed distribution, so their distribution names are unknown rather than merely undeclared.`,
+      severity: 'info',
+    });
   }
   if (imports && !imports.stdlibAvailable) {
     notes.push({
@@ -389,7 +330,14 @@ export function planDependencies(
     misplaced,
     unused,
     drift,
-    providerMappingReliable: !(imports?.providersUnavailable ?? true),
+    // A partial mapping is disclosed through `unmappedImports`; the claim here is
+    // only that the interpreter could see an installed environment at all. When
+    // it could see one yet owned none of the project's imports, it is not the
+    // project's interpreter and nothing it reports should be trusted.
+    providerMappingReliable:
+      !(imports?.providersUnavailable ?? true) &&
+      !(thirdPartyCount > 0 && unmappedImports === thirdPartyCount),
+    unmappedImports,
     unparsable: imports?.unparsable ?? [],
     warnings,
     notes,

package/src/environment/discovery.ts

@@ -4,13 +4,19 @@ import { findProjectRoot, findVenvDir, isFile } from '../project/root.ts';
 import {
   type EnvironmentSection,
   type LockPackage,
-  resolveInterpreter,
+  resolveProjectInterpreter,
   runScanProject,
 } from '../project/scanner.ts';
 import { inspectTools, type ToolAvailability } from './tools.ts';
 
 export interface PythonEnvironment {
   interpreter?: string;
+  /**
+   * Whether the analysed interpreter belongs to the project environment or was
+   * taken from PATH. Analysis facts such as `site-packages` contents and the
+   * reported version are only the project's when this is `venv`.
+   */
+  interpreterOrigin?: 'venv' | 'path';
   python?: EnvironmentSection;
   projectRoot?: string;
   venvDir?: string;
@@ -57,12 +63,13 @@ export async function detectPythonEnvironment(
 ): Promise<PythonEnvironment> {
   const warnings: Diagnostic[] = [];
   const suggestions: string[] = [];
-  const interpreter = await resolveInterpreter(cwd, signal);
   const projectRoot = await findProjectRoot(cwd);
   const scanRoot = projectRoot ?? cwd;
 
   let python: EnvironmentSection | undefined;
   let lockPackages: LockPackage[] = [];
+  const resolved = await resolveProjectInterpreter(scanRoot, cwd, signal);
+  const interpreter = resolved.interpreter;
   if (interpreter) {
     const scan = await runScanProject(
       cwd,
@@ -79,7 +86,7 @@ export async function detectPythonEnvironment(
     warnings.push(
       warn(
         'PYTHON_NOT_FOUND',
-        'No Python 3 interpreter was found on PATH; every analysis tool degrades to static inspection only.',
+        'No Python 3 interpreter was found in the project environment or on PATH; every analysis tool degrades to static inspection only.',
       ),
     );
     suggestions.push('Install Python 3.11 or newer so pyproject.toml and uv.lock can be parsed.');
@@ -101,6 +108,25 @@ export async function detectPythonEnvironment(
   const lockPresent = projectRoot ? await isFile(`${projectRoot}/uv.lock`) : false;
   const tools = await inspectTools({ venvDir, lockPackages });
 
+  // A distribution recorded in the lockfile is installable, not installed. When
+  // the project environment exists but a declared tool has no console script,
+  // the environment was synced without it and every later step that needs it
+  // will fail, so this is reported rather than left to be discovered later.
+  const missingDeclaredTools = venvDir ? tools.filter((tool) => tool.installable) : [];
+  if (missingDeclaredTools.length > 0) {
+    const names = missingDeclaredTools.map((tool) => tool.name).join(', ');
+    warnings.push(
+      warn(
+        'TOOL_NOT_INSTALLED',
+        `${names} ${missingDeclaredTools.length === 1 ? 'is' : 'are'} recorded in uv.lock but has no executable in .venv, so it cannot be run right now.`,
+        venvDir,
+      ),
+    );
+    suggestions.push(
+      'Run uv sync --frozen --all-groups --all-extras: a plain uv sync removes extras declared in [project.optional-dependencies].',
+    );
+  }
+
   const uvVersion = await toolVersion(cwd, 'uv', signal);
   if (!uvVersion) {
     warnings.push(
@@ -155,6 +181,7 @@ export async function detectPythonEnvironment(
 
   return {
     interpreter,
+    interpreterOrigin: resolved.origin,
     python,
     projectRoot,
     venvDir,

package/src/environment/tools.ts

@@ -22,10 +22,20 @@ export type ToolVersionSource = 'lock' | 'cli' | 'unknown';
 
 export interface ToolAvailability {
   name: string;
-  /** True when the tool is runnable, either locally or through `uv run`. */
+  /**
+   * True only when a runnable executable was found right now.
+   *
+   * A distribution recorded in `uv.lock` is *installable*, not available: the
+   * environment may have had it removed. Reporting it as available made a
+   * broken virtual environment look healthy.
+   */
   available: boolean;
   /** True when `uv.lock` records the distribution, so `uv sync` can install it. */
   declared: boolean;
+  /** True when `uv sync` would provide the tool that is not runnable yet. */
+  installable: boolean;
+  /** True when the console script was found inside the project environment. */
+  installed: boolean;
   /** Absolute path when an executable was found. */
   executable?: string;
   /** Where the executable was found: the project environment or the host PATH. */
@@ -128,8 +138,10 @@ export async function inspectTools(input: ToolProbeInput = {}): Promise<ToolAvai
 
       return {
         name,
-        available: Boolean(executable) || lockVersion !== undefined,
+        available: Boolean(executable),
         declared: lockVersion !== undefined,
+        installable: lockVersion !== undefined && !executable,
+        installed: venvScript !== undefined,
         executable,
         origin,
         version: lockVersion,
@@ -139,3 +151,30 @@ export async function inspectTools(input: ToolProbeInput = {}): Promise<ToolAvai
     }),
   );
 }
+
+export interface RequiredToolCheck {
+  /** False when there was no project environment to inspect at all. */
+  checked: boolean;
+  venvDir?: string;
+  /** Names with no console script in the project environment. */
+  missing: string[];
+}
+
+/**
+ * Confirm that the tools a later step depends on are runnable *now*.
+ *
+ * `uv sync` can legitimately finish with exit code 0 while removing the very
+ * tools the next step needs, so the environment is re-checked between the two
+ * instead of trusting the exit code.
+ */
+export async function checkRequiredTools(
+  venvDir: string | undefined,
+  names: readonly string[],
+): Promise<RequiredToolCheck> {
+  if (!venvDir) return { checked: false, missing: [] };
+  const missing: string[] = [];
+  for (const name of names) {
+    if (!(await findVenvScript(venvDir, name))) missing.push(name);
+  }
+  return { checked: true, venvDir, missing };
+}