postgresai 0.16.0-dev.1 → 0.16.0-dev.11

package/CHANGELOG.md

@@ -0,0 +1,11 @@
+# Changelog
+
+## Unreleased
+
+### Fixed
+
+- `checkup --markdown` previously performed server-side conversion and sent the
+  full report JSON to the PostgresAI API even when `--no-upload` was set. The
+  flags are now mutually exclusive, and `--no-upload` prevents report data from
+  being sent to the PostgresAI API. Use `--json` or `--output` for local-only
+  output.

package/bin/postgres-ai.ts

@@ -13,6 +13,16 @@ import { Client } from "pg";
 import { startMcpServer } from "../lib/mcp-server";
 import { fetchIssues, fetchIssueComments, createIssueComment, fetchIssue, createIssue, updateIssue, updateIssueComment, fetchActionItem, fetchActionItems, createActionItem, updateActionItem, type ConfigChange } from "../lib/issues";
 import { fetchReports, fetchAllReports, fetchReportFiles, fetchReportFileData, renderMarkdownForTerminal, parseFlexibleDate } from "../lib/reports";
+import {
+  executeJoeCommand,
+  listProjects,
+  getCommandOutput,
+  formatJoeOutput,
+  formatProjectsTable,
+  DEFAULT_BUDGET_MS,
+  type JoeCommand,
+  type ExecuteJoeOutcome,
+} from "../lib/joe";
 import { resolveBaseUrls } from "../lib/util";
 import { registerAasCollection, parseVcpus } from "../lib/aas-onboard";
 import { uploadFile, downloadFile, buildMarkdownLink, uploadAttachments, appendAttachmentsToContent } from "../lib/storage";
@@ -335,12 +345,18 @@ function prepareUploadConfig(
       console.error("Tip: run 'postgresai auth' or pass --api-key / set PGAI_API_KEY");
       return null; // Signal to exit
     }
-    // No credentials and upload not explicitly requested: fall back to
-    // local-only mode, but say so prominently — skipping the upload silently
-    // hides the fact that results never reach the Console.
-    console.error("Notice: no API key configured — results will NOT be uploaded to PostgresAI.");
-    console.error("        To upload: run 'postgresai auth login' or pass --api-key / set PGAI_API_KEY.");
-    console.error("        To run locally without this notice, pass --no-upload.");
+    if (opts.markdown) {
+      console.error("Notice: no API key configured — regular report upload is disabled.");
+      console.error("        The full report JSON will still be sent to the PostgresAI API for markdown conversion.");
+      console.error("        To avoid sending report data, replace --markdown with --no-upload and --json or --output.");
+    } else {
+      // No credentials and upload not explicitly requested: fall back to
+      // local-only mode, but say so prominently — skipping the upload silently
+      // hides the fact that results never reach the Console.
+      console.error("Notice: no API key configured — results will NOT be uploaded to PostgresAI.");
+      console.error("        To upload: run 'postgresai auth login' or pass --api-key / set PGAI_API_KEY.");
+      console.error("        To run locally without this notice, pass --no-upload.");
+    }
     return undefined; // Skip upload, run checks locally
   }
 
@@ -1996,7 +2012,7 @@ program
     "project name or ID for remote upload (used with --upload; defaults to config defaultProject; auto-generated on first run)"
   )
   .option("--json", "output JSON to stdout")
-  .option("--markdown", "output markdown to stdout")
+  .option("--markdown", "output markdown via PostgresAI API (transmits the full report JSON)")
   .addHelpText(
     "after",
     [
@@ -2010,7 +2026,7 @@ program
       "  postgresai checkup postgresql://user:pass@host:5432/db --check-id H002",
       "  postgresai checkup postgresql://user:pass@host:5432/db --output ./reports",
       "  postgresai checkup postgresql://user:pass@host:5432/db --no-upload --json",
-      "  postgresai checkup postgresql://user:pass@host:5432/db --no-upload --markdown",
+      "  postgresai checkup postgresql://user:pass@host:5432/db --markdown",
     ].join("\n")
   )
   .action(async (checkIdOrConn: string | undefined, connArg: string | undefined, opts: CheckupOptions, cmd: Command) => {
@@ -2060,9 +2076,14 @@ program
       process.exitCode = 1;
       return;
     }
-    // Note: --json, --markdown and --upload/--no-upload are independent flags.
-    // Use --no-upload to explicitly disable upload when using --json or --markdown.
     const uploadExplicitlyDisabled = opts.upload === false;
+    if (uploadExplicitlyDisabled && shouldConvertMarkdown) {
+      console.error("Error: --no-upload and --markdown are mutually exclusive");
+      console.error("Markdown conversion is performed by the PostgresAI API and transmits the full report JSON.");
+      console.error("Drop --no-upload to allow transmission, or use --json or --output for local-only output.");
+      process.exitCode = 1;
+      return;
+    }
     let shouldUpload = !uploadExplicitlyDisabled;
 
     // Preflight: validate/create output directory BEFORE connecting / running checks.
@@ -2308,7 +2329,9 @@ program
 
         console.log('\nFor details:');
         console.log('  --json          Output JSON');
-        console.log('  --markdown      Output markdown');
+        if (!uploadExplicitlyDisabled) {
+          console.log('  --markdown      Output markdown via PostgresAI API');
+        }
         console.log('  --output <dir>  Save to directory');
       }
     } catch (error) {
@@ -2443,9 +2466,44 @@ interface MonitoringRegistration {
  *
  * Never throws — registration is best-effort; returns null on failure.
  */
+
+/**
+ * Classify how `mon local-install` should register the monitoring instance,
+ * given the raw `--project` value and the resolved instance id.
+ *
+ * - With an instance id: ADOPT the provisioned instance. No project name is
+ *   required (the platform returns the real project); a provided name is
+ *   normalized and carried through for messaging/fallback.
+ * - No instance id and no project name: ERROR. The hardcoded
+ *   "postgres-ai-monitoring" default was removed, and a nameless legacy
+ *   self-registration is rejected by v1.monitoring_instance_register (PT400).
+ * - No instance id but a project name: legacy SELF-REGISTER.
+ *
+ * Pure (no I/O) so the decision is unit-testable independently of the large
+ * install command. `projectName` is the trimmed value, or undefined when empty.
+ */
+type MonRegistrationPlan =
+  | { kind: "adopt"; projectName: string | undefined }
+  | { kind: "self-register"; projectName: string }
+  | { kind: "error-missing-project"; projectName: undefined };
+
+function planMonitoringRegistration(args: {
+  project?: string;
+  instanceId?: string;
+}): MonRegistrationPlan {
+  const projectName = args.project?.trim() || undefined;
+  if (args.instanceId) {
+    return { kind: "adopt", projectName };
+  }
+  if (!projectName) {
+    return { kind: "error-missing-project", projectName: undefined };
+  }
+  return { kind: "self-register", projectName };
+}
+
 async function registerMonitoringInstance(
   apiKey: string,
-  projectName: string,
+  projectName: string | undefined,
   opts?: { apiBaseUrl?: string; debug?: boolean; instanceId?: string; retries?: number; retryDelayMs?: number }
 ): Promise<MonitoringRegistration | null> {
   const { apiBaseUrl } = resolveBaseUrls(opts);
@@ -2457,16 +2515,25 @@ async function registerMonitoringInstance(
   // moment to recover; skipped before the first attempt. Tests pass 0.
   const retryDelayMs = opts?.retryDelayMs ?? 400;
 
+  // Omit project_name entirely when empty: the adopt path (instance_id present)
+  // relies on the platform returning the real project, and a nameless legacy
+  // self-registration is rejected by v1.monitoring_instance_register (PT400).
+  const hasProjectName = !!(projectName && projectName.trim());
+
   if (debug) {
     console.error(`\nDebug: Registering monitoring instance...`);
     console.error(`Debug: POST ${url}`);
-    console.error(`Debug: project_name=${projectName}${instanceId ? ` instance_id=${instanceId}` : ""}`);
+    console.error(
+      `Debug: ${hasProjectName ? `project_name=${projectName}` : "project_name=(omitted)"}${instanceId ? ` instance_id=${instanceId}` : ""}`
+    );
   }
 
   const requestBody: Record<string, string> = {
     api_token: apiKey,
-    project_name: projectName,
   };
+  if (hasProjectName) {
+    requestBody.project_name = projectName as string;
+  }
   if (instanceId) {
     requestBody.instance_id = instanceId;
   }
@@ -3137,8 +3204,11 @@ mon
     // and persisted; the legacy self-registration stays fire-and-forget
     // (issue platform-all#311).
     if (apiKey && !opts.demo) {
-      const projectName = opts.project || "postgres-ai-monitoring";
       const instanceId = opts.instanceId || process.env.PGAI_INSTANCE_ID;
+      const plan = planMonitoringRegistration({ project: opts.project, instanceId });
+      const projectName = plan.projectName;
+      // `instanceId` truthy ⟺ plan.kind === "adopt"; branch on it directly so
+      // TypeScript narrows instanceId to a defined string in the adopt path.
       if (instanceId) {
         const reg = await registerMonitoringInstance(apiKey, projectName, {
           apiBaseUrl: globalOpts.apiBaseUrl,
@@ -3160,11 +3230,17 @@ mon
           // Request succeeded but carried no usable project field — don't claim
           // adoption, but don't report a hard failure either (no re-run needed).
           console.error(
-            `⚠ Adopted provisioned instance ${instanceId} but the platform returned no project — reports will use project '${projectName}'`
+            `⚠ Adopted provisioned instance ${instanceId} but the platform returned no project` +
+              (projectName
+                ? ` — reports will use project '${projectName}'`
+                : ` — reports will have no project until 'postgresai mon local-install' is re-run with --project <name>`)
           );
         } else {
           console.error(
-            `⚠ Could not adopt provisioned instance ${instanceId} — reports will use project '${projectName}' until 'postgresai mon local-install' is re-run`
+            `⚠ Could not adopt provisioned instance ${instanceId}` +
+              (projectName
+                ? ` — reports will use project '${projectName}' until 'postgresai mon local-install' is re-run`
+                : ` — reports will have no project until 'postgresai mon local-install' is re-run with --project <name>`)
           );
         }
 
@@ -3188,6 +3264,17 @@ mon
             `⚠ AAS auto-collection not registered (${aas.reason}); it can be enabled later by re-running 'postgresai mon local-install'\n`
           );
         }
+      } else if (plan.kind === "error-missing-project") {
+        // Legacy self-registration (no --instance-id) now requires a project
+        // name: the hardcoded "postgres-ai-monitoring" default was removed, and
+        // v1.monitoring_instance_register raises PT400 for a nameless legacy
+        // registration. Console-provisioned installs should adopt with
+        // --instance-id instead.
+        console.error(
+          "✗ A project name is required for self-registration (the 'postgres-ai-monitoring' default was removed). " +
+            "Re-run with --project <name>, or adopt a console-provisioned instance with --instance-id <uuid>."
+        );
+        process.exitCode = 1;
       } else {
         void registerMonitoringInstance(apiKey, projectName, {
           apiBaseUrl: globalOpts.apiBaseUrl,
@@ -3867,13 +3954,9 @@ targets
 // Authentication and API key management
 const auth = program.command("auth").description("authentication and API key management");
 
-auth
-  .command("login", { isDefault: true })
-  .description("authenticate via browser (OAuth) or store API key directly")
-  .option("--set-key <key>", "store API key directly without OAuth flow")
-  .option("--port <port>", "local callback server port (default: random)", parseInt)
-  .option("--debug", "enable debug output")
-  .action(async (opts: { setKey?: string; port?: number; debug?: boolean }) => {
+type AuthLoginOptions = { setKey?: string; port?: number; debug?: boolean };
+
+async function runAuthLogin(opts: AuthLoginOptions) {
     // If --set-key is provided, store it directly without OAuth
     if (opts.setKey) {
       const trimmedKey = opts.setKey.trim();
@@ -4127,7 +4210,21 @@ auth
       console.error(`Authentication error: ${message}`);
       process.exit(1);
     }
-  });
+}
+
+function configureLoginCommand(command: Command): Command {
+  return command
+    .description("authenticate via browser (OAuth) or store API key directly")
+    .option("--set-key <key>", "store API key directly without OAuth flow")
+    .option("--port <port>", "local callback server port (default: random)", parseInt)
+    .option("--debug", "enable debug output");
+}
+
+configureLoginCommand(auth.command("login", { isDefault: true }))
+  .action(runAuthLogin);
+
+configureLoginCommand(program.command("login"))
+  .action(runAuthLogin);
 
 auth
   .command("show-key")
@@ -5312,6 +5409,291 @@ function tryParseJson(s: string): unknown {
   try { return JSON.parse(s); } catch { return s; }
 }
 
+// ---------------------------------------------------------------------------
+// Joe command surface (Joe API v2 CLI v1, issue #438). Every verb builds the
+// RAW command text Joe's /webui/command dispatches (`plan <sql>`, `\d users`),
+// runs it via the synchronous `v1.joe_command_run`, and polls
+// `v1.joe_command_output` until `ok`/`error` within the one-shot budget.
+// `--project` accepts a numeric id OR an alias/name/label. The backend rpcs
+// are mocked in tests.
+// ---------------------------------------------------------------------------
+
+interface JoeCliOpts {
+  project?: string;
+  instanceId?: string;
+  budget?: number;
+  variant?: string;
+  debug?: boolean;
+  json?: boolean;
+}
+
+function printJoeOutcome(outcome: ExecuteJoeOutcome, json: boolean, budgetMs?: number): void {
+  // The expiry hint reports the ACTUAL effective budget (--budget when given,
+  // the default otherwise) — not a hardcoded DEFAULT_BUDGET_MS.
+  const effectiveBudgetMs =
+    typeof budgetMs === "number" && Number.isFinite(budgetMs) ? budgetMs : DEFAULT_BUDGET_MS;
+  const budgetSeconds = Math.round(effectiveBudgetMs / 1000);
+  // One-shot budget reached before a terminal state — hand back a resume handle.
+  // This is expected (a cold clone), NOT a failure: exit 0.
+  if (outcome.budgetExpired) {
+    if (json) {
+      console.log(
+        JSON.stringify(
+          {
+            command_id: outcome.commandId,
+            status: outcome.status,
+            budget_expired: true,
+            resume: `pgai joe result ${outcome.commandId}`,
+          },
+          null,
+          2
+        )
+      );
+    } else {
+      console.log(
+        `started ${outcome.commandId} · ${outcome.status} · budget ${budgetSeconds}s reached — resume:  pgai joe result ${outcome.commandId}`
+      );
+    }
+    return;
+  }
+
+  const output = outcome.output;
+  if (outcome.status === "ok") {
+    if (!output) {
+      console.error(`command ${outcome.commandId} ok but output is empty`);
+      process.exitCode = 1;
+      return;
+    }
+    if (json) console.log(JSON.stringify(output, null, 2));
+    else {
+      console.log(`command ${outcome.commandId} · ok`);
+      const body = formatJoeOutput(output);
+      if (body) console.log(body);
+    }
+    return;
+  }
+
+  // Terminal error.
+  if (json && output) console.log(JSON.stringify(output, null, 2));
+  console.error(`command ${outcome.commandId} error: ${output?.error ?? "command failed"}`);
+  process.exitCode = 1;
+}
+
+async function runJoeCli(
+  command: JoeCommand,
+  arg: string | null,
+  opts: JoeCliOpts
+): Promise<void> {
+  try {
+    const rootOpts = program.opts<CliOptions>();
+    const cfg = config.readConfig();
+    const { apiKey } = getConfig(rootOpts);
+    if (!apiKey) {
+      console.error("API key is required. Run 'pgai auth' first or set --api-key.");
+      process.exitCode = 1;
+      return;
+    }
+    const projectRef = (opts.project ?? cfg.defaultProject ?? "").toString().trim();
+    const instanceRef = (opts.instanceId ?? "").toString().trim();
+    if (!projectRef && !instanceRef) {
+      console.error(
+        "Specify --instance-id <id> (or --project <id|alias> once projects_list is available)."
+      );
+      process.exitCode = 1;
+      return;
+    }
+    const { apiBaseUrl } = resolveBaseUrls(rootOpts, cfg);
+    if (typeof opts.budget === "number" && (!Number.isFinite(opts.budget) || opts.budget < 0)) {
+      throw new Error("--budget must be a non-negative number of seconds");
+    }
+    const budgetMs = typeof opts.budget === "number" ? opts.budget * 1000 : undefined;
+
+    const outcome = await executeJoeCommand({
+      apiKey,
+      apiBaseUrl,
+      command,
+      project: projectRef || undefined,
+      instanceId: instanceRef || undefined,
+      input: { arg, variant: opts.variant ?? null },
+      orgId: cfg.orgId ?? undefined,
+      budgetMs,
+      debug: !!opts.debug,
+    });
+
+    printJoeOutcome(outcome, !!opts.json, budgetMs);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.error(message);
+    process.exitCode = 1;
+  }
+}
+
+/** Attach the shared Joe options (--instance-id / --project / --budget / --debug / --json). */
+function withJoeOptions(cmd: import("commander").Command): import("commander").Command {
+  return cmd
+    .option(
+      "--instance-id <id>",
+      "target the Joe instance id directly (skips --project resolution; the v1 path while projects_list is unavailable)"
+    )
+    .option("--project <id|alias>", "target project by numeric id OR alias/name (requires projects_list)")
+    .option("--budget <seconds>", "one-shot poll budget in seconds (default 25)", (v) => parseFloat(v))
+    .option("--debug", "enable debug output")
+    .option("--json", "output raw JSON");
+}
+
+const joe = program
+  .command("joe")
+  .description("Joe — plan/EXPLAIN/exec queries on ephemeral DBLab clones");
+
+withJoeOptions(
+  joe
+    .command("plan <sql>")
+    .description("plan a query (EXPLAIN, plan-only — no execution; the fast/safe default)")
+).action(async (sql: string, opts: JoeCliOpts) => {
+  await runJoeCli("plan", sql, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("explain <sql>")
+    .description("EXPLAIN + EXPLAIN ANALYZE a query (EXECUTES on the ephemeral clone)")
+).action(async (sql: string, opts: JoeCliOpts) => {
+  await runJoeCli("explain", sql, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("exec <sql>")
+    .description("run arbitrary DDL/DML on the clone (e.g. create index, analyze)")
+).action(async (sql: string, opts: JoeCliOpts) => {
+  await runJoeCli("exec", sql, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("hypo <args>")
+    .description("HypoPG hypothetical indexes (e.g. `hypo create index on users (email)`, `hypo desc`, `hypo reset`)")
+).action(async (args: string, opts: JoeCliOpts) => {
+  await runJoeCli("hypo", args, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("activity")
+    .description("running-activity snapshot (pg_stat_activity) on the clone")
+).action(async (opts: JoeCliOpts) => {
+  await runJoeCli("activity", null, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("terminate <pid>")
+    .description("pg_terminate_backend(pid) on the clone")
+).action(async (pid: string, opts: JoeCliOpts) => {
+  // The bare-positive-integer pid guard lives in buildJoeCommandText, which
+  // runs before any network call — a mistyped pid never reaches an rpc.
+  await runJoeCli("terminate", pid, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("reset")
+    .description("reset/recreate the session's thin clone")
+).action(async (opts: JoeCliOpts) => {
+  await runJoeCli("reset", null, opts);
+});
+
+withJoeOptions(
+  joe
+    .command("describe <object>")
+    .description("\\d-family schema/relation/index metadata")
+    .option("--variant <variant>", "\\d-family variant (e.g. \\d+, \\di, \\dt)")
+).action(async (object: string, opts: JoeCliOpts) => {
+  await runJoeCli("describe", object, opts);
+});
+
+// Resume / inspect a previously started command by id (the budget-expiry
+// resume handle printed by the one-shot verbs).
+joe
+  .command("result <commandId>")
+  .description("fetch a Joe command's output by id (resume a budget-expired one-shot)")
+  .option("--debug", "enable debug output")
+  .option("--json", "output raw JSON")
+  .action(async (commandId: string, opts: { debug?: boolean; json?: boolean }) => {
+    try {
+      const rootOpts = program.opts<CliOptions>();
+      const cfg = config.readConfig();
+      const { apiKey } = getConfig(rootOpts);
+      if (!apiKey) {
+        console.error("API key is required. Run 'pgai auth' first or set --api-key.");
+        process.exitCode = 1;
+        return;
+      }
+      const { apiBaseUrl } = resolveBaseUrls(rootOpts, cfg);
+      const output = await getCommandOutput({ apiKey, apiBaseUrl, commandId, debug: !!opts.debug });
+      if (opts.json) {
+        console.log(JSON.stringify(output, null, 2));
+        // Same exit contract as the human output and the one-shot path
+        // (printJoeOutcome): only `ok` is a result — scripts consuming --json
+        // must not proceed on `pending`/`error`.
+        if (output.status !== "ok") {
+          process.exitCode = 1;
+        }
+        return;
+      }
+      if (output.status === "ok") {
+        console.log(`command ${output.command_id} · ok`);
+        const body = formatJoeOutput(output);
+        if (body) console.log(body);
+      } else if (output.status === "error") {
+        console.error(`command ${output.command_id} error: ${output.error ?? "command failed"}`);
+        process.exitCode = 1;
+      } else {
+        console.error(`command ${output.command_id} · ${output.status} — result is not ready`);
+        process.exitCode = 1;
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.error(message);
+      process.exitCode = 1;
+    }
+  });
+
+// Org-level discovery — a general postgresai command, NOT a Joe endpoint.
+program
+  .command("projects")
+  .description("list the org's projects (shows which have Joe ready) — org-level, not a Joe endpoint")
+  .option("--debug", "enable debug output")
+  .option("--json", "output raw JSON")
+  .action(async (opts: { debug?: boolean; json?: boolean }) => {
+    try {
+      const rootOpts = program.opts<CliOptions>();
+      const cfg = config.readConfig();
+      const { apiKey } = getConfig(rootOpts);
+      if (!apiKey) {
+        console.error("API key is required. Run 'pgai auth' first or set --api-key.");
+        process.exitCode = 1;
+        return;
+      }
+      const { apiBaseUrl } = resolveBaseUrls(rootOpts, cfg);
+      const projects = await listProjects({
+        apiKey,
+        apiBaseUrl,
+        orgId: cfg.orgId ?? undefined,
+        debug: !!opts.debug,
+      });
+      if (opts.json) {
+        console.log(JSON.stringify(projects, null, 2));
+      } else {
+        console.log(formatProjectsTable(projects));
+      }
+    } catch (err) {
+      const message = err instanceof Error ? err.message : String(err);
+      console.error(message);
+      process.exitCode = 1;
+    }
+  });
+
 // MCP server
 const mcp = program.command("mcp").description("MCP server integration");
 
@@ -5488,3 +5870,4 @@ if (import.meta.main) {
 // same functions used by the `mon` commands).
 export { refreshBundledComposeIfStale, readDeployedTag, isValidComposeYaml };
 export { registerMonitoringInstance, resolveAdoptedProject, type MonitoringRegistration };
+export { planMonitoringRegistration, type MonRegistrationPlan };