wellmarked 1.1.0

package/src/client.ts

@@ -0,0 +1,463 @@
+/**
+ * WellMarked client.
+ *
+ * The client is a thin, typed wrapper around the HTTP API. All endpoint
+ * methods are async — there is no separate sync/async split as in the
+ * Python SDK because JavaScript I/O is async by default.
+ *
+ *     import { WellMarked } from "wellmarked";
+ *
+ *     const wm = new WellMarked({ apiKey: "wm_..." });
+ *     const result = await wm.extract("https://example.com/article");
+ *     console.log(result.markdown);
+ *
+ * The API key can also be passed via the `WELLMARKED_API_KEY` environment
+ * variable (Node.js), in which case `new WellMarked()` is enough.
+ */
+import {
+  APIConnectionError,
+  APIStatusError,
+  WellMarkedError,
+  fromResponse,
+} from "./errors.js";
+import {
+  type BulkJob,
+  type CrawlJob,
+  type ExtractResult,
+  type RotatedKey,
+  type Usage,
+  bulkJobFromResponse,
+  crawlJobFromResponse,
+  extractResultFromResponse,
+  rotatedKeyFromResponse,
+  usageFromResponse,
+} from "./models.js";
+import { VERSION } from "./version.js";
+
+const DEFAULT_BASE_URL = "https://api.wellmarked.io";
+const DEFAULT_TIMEOUT_MS = 30_000;
+
+const RESERVED_HEADERS = new Set([
+  "authorization",
+  "content-type",
+  "accept",
+]);
+
+export interface WellMarkedOptions {
+  /**
+   * Your WellMarked API key (`wm_...`). Falls back to the
+   * `WELLMARKED_API_KEY` environment variable (Node.js only).
+   */
+  apiKey?: string;
+  /** API base URL. Override for testing. */
+  baseUrl?: string;
+  /** Per-request timeout, milliseconds. Defaults to 30000 (30s). */
+  timeoutMs?: number;
+  /**
+   * Bring your own `fetch`. Defaults to the global `fetch`. Useful for
+   * polyfills, custom agents/proxies, or test mocking.
+   */
+  fetch?: typeof fetch;
+  /**
+   * Extra headers sent on every request — useful for adding an internal
+   * correlation id, a custom user agent suffix, etc.
+   *
+   * Authorization / Content-Type / Accept are reserved and silently
+   * ignored if passed (the SDK manages those itself).
+   */
+  headers?: Record<string, string>;
+}
+
+export interface ExtractOptions {
+  /**
+   * Use Playwright to render JS-heavy pages. Requires a Pro/Enterprise
+   * plan AND `ENABLE_JS_RENDERING=true` on the API instance.
+   */
+  renderJs?: boolean;
+}
+
+export interface BulkOptions {
+  renderJs?: boolean;
+}
+
+export interface CrawlOptions {
+  /** Max BFS depth from the root. Defaults to 1. Must be >= 0. */
+  depth?: number;
+  renderJs?: boolean;
+}
+
+export interface WaitForJobOptions {
+  /** Milliseconds to sleep between polls. Defaults to 2000. */
+  pollIntervalMs?: number;
+  /** Total ms to wait before timing out. `null` waits forever. Defaults to 300000 (5 min). */
+  timeoutMs?: number | null;
+}
+
+function resolveApiKey(apiKey: string | undefined): string {
+  if (apiKey) return apiKey;
+  const env =
+    typeof process !== "undefined" && process.env
+      ? process.env.WELLMARKED_API_KEY
+      : undefined;
+  if (env) return env;
+  throw new Error(
+    "No API key provided. Pass apiKey: ... to the client or set the " +
+      "WELLMARKED_API_KEY environment variable. Generate a key at " +
+      "https://wellmarked.io.",
+  );
+}
+
+function defaultHeaders(apiKey: string): Record<string, string> {
+  return {
+    Authorization: `Bearer ${apiKey}`,
+    "Content-Type": "application/json",
+    Accept: "application/json",
+    "User-Agent": `wellmarked-js/${VERSION}`,
+  };
+}
+
+function mergeHeaders(
+  apiKey: string,
+  extra: Record<string, string> | undefined,
+): Record<string, string> {
+  const out = defaultHeaders(apiKey);
+  if (!extra) return out;
+  for (const [k, v] of Object.entries(extra)) {
+    if (RESERVED_HEADERS.has(k.toLowerCase())) continue;
+    out[k] = v;
+  }
+  return out;
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+interface RequestInitWithSignal {
+  method: string;
+  headers: Record<string, string>;
+  body?: string;
+  signal?: AbortSignal;
+}
+
+export class WellMarked {
+  private apiKey: string;
+  private readonly baseUrl: string;
+  private readonly timeoutMs: number;
+  private readonly fetchImpl: typeof fetch;
+  private readonly extraHeaders: Record<string, string>;
+
+  constructor(options: WellMarkedOptions = {}) {
+    this.apiKey = resolveApiKey(options.apiKey);
+    this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+    this.timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+    const f = options.fetch ?? (typeof fetch !== "undefined" ? fetch : undefined);
+    if (!f) {
+      throw new Error(
+        "No fetch implementation available. Pass `fetch:` to the client " +
+          "(undici, node-fetch, etc.) or upgrade to Node 18+.",
+      );
+    }
+    // Bind so `this` isn't lost when calling globalThis.fetch.
+    this.fetchImpl = f.bind(globalThis) as typeof fetch;
+    this.extraHeaders = {};
+    if (options.headers) {
+      for (const [k, v] of Object.entries(options.headers)) {
+        if (RESERVED_HEADERS.has(k.toLowerCase())) continue;
+        this.extraHeaders[k] = v;
+      }
+    }
+  }
+
+  // ── Endpoints ──────────────────────────────────────────────────────────────
+
+  /**
+   * Extract clean Markdown from a single URL.
+   *
+   * Throws:
+   *   - `RateLimitError`            — monthly plan limit reached.
+   *   - `UnprocessableEntityError`  — `no_content`, `target_timeout`, or
+   *                                   `js_rendering_disabled`.
+   *   - `AuthenticationError`       — missing or invalid API key.
+   */
+  async extract(url: string, options: ExtractOptions = {}): Promise<ExtractResult> {
+    const body = await this.request("POST", "/extract", {
+      url,
+      render_js: options.renderJs === true,
+    });
+    return extractResultFromResponse(body as Record<string, unknown>);
+  }
+
+  /**
+   * Submit a batch of URLs for concurrent extraction.
+   *
+   * Returns immediately with `status="queued"`. Poll with `getJob` or
+   * block with `waitForJob` to collect results.
+   *
+   * Throws:
+   *   - `PermissionDeniedError`     — `plan_not_supported` (Free tier).
+   *   - `UnprocessableEntityError`  — `bulk_cap_exceeded` (50 on Pro).
+   *   - `RateLimitError`            — would exceed remaining monthly quota.
+   */
+  async bulk(urls: Iterable<string>, options: BulkOptions = {}): Promise<BulkJob> {
+    const urlList = Array.from(urls);
+    if (urlList.length === 0) {
+      throw new Error("bulk() requires at least one URL.");
+    }
+    const body = await this.request("POST", "/bulk", {
+      urls: urlList,
+      render_js: options.renderJs === true,
+    });
+    return bulkJobFromResponse(body as Record<string, unknown>);
+  }
+
+  /**
+   * Polymorphic job lookup — works for both bulk and crawl jobs.
+   *
+   * Calls `GET /bulk/{jobId}` first, then inspects the response's `kind`
+   * discriminator field. If the job is actually a crawl, a second request
+   * to `GET /crawl/{jobId}` fetches the full crawl shape (with per-item
+   * depth and the truncated flags). Returns `BulkJob` or `CrawlJob`
+   * accordingly.
+   *
+   * Use `isCrawlJob(job)` (or check `job.kind === "crawl"`) to branch on
+   * crawl-specific behavior. The shared interface (`status`, `completed`,
+   * `total`, `results`, `done`) works on either type.
+   *
+   * Jobs are retained for 6 hours after completion.
+   */
+  async getJob(jobId: string): Promise<BulkJob | CrawlJob> {
+    const body = (await this.request("GET", `/bulk/${jobId}`)) as Record<
+      string,
+      unknown
+    >;
+    // /bulk/{id} answers for any jobId today (the endpoint just serializes
+    // results in the bulk shape regardless of stored job_type). The `kind`
+    // field tells us whether we got a bulk-shaped response of a crawl
+    // job; if so, re-fetch via /crawl/{id} for the proper shape.
+    if (body.kind === "crawl") {
+      const crawlBody = (await this.request("GET", `/crawl/${jobId}`)) as Record<
+        string,
+        unknown
+      >;
+      return crawlJobFromResponse(crawlBody);
+    }
+    return bulkJobFromResponse(body);
+  }
+
+  /**
+   * Block until a job reaches `status="done"` (or timeout). Works for both
+   * bulk and crawl jobs.
+   *
+   * The first call uses the polymorphic `getJob` to discover the job's
+   * kind. Subsequent polls go directly to the typed endpoint, so a crawl
+   * job only pays the dispatch round-trip once.
+   *
+   * Throws:
+   *   - `Error` with message "did not finish within ..." — the job didn't
+   *     finish before `timeoutMs` elapsed.
+   */
+  async waitForJob(
+    jobId: string,
+    options: WaitForJobOptions = {},
+  ): Promise<BulkJob | CrawlJob> {
+    const pollIntervalMs = options.pollIntervalMs ?? 2_000;
+    const timeoutMs = options.timeoutMs === undefined ? 300_000 : options.timeoutMs;
+    const deadline = timeoutMs === null ? null : Date.now() + timeoutMs;
+
+    let job: BulkJob | CrawlJob = await this.getJob(jobId);
+    const isCrawl = job.kind === "crawl";
+
+    while (!job.done) {
+      if (deadline !== null && Date.now() >= deadline) {
+        throw new Error(
+          `Job ${jobId} did not finish within ${timeoutMs}ms ` +
+            `(last status: ${job.status}, ${job.completed}/${job.total})`,
+        );
+      }
+      await sleep(pollIntervalMs);
+      const path = isCrawl ? `/crawl/${jobId}` : `/bulk/${jobId}`;
+      const body = (await this.request("GET", path)) as Record<string, unknown>;
+      job = isCrawl ? crawlJobFromResponse(body) : bulkJobFromResponse(body);
+    }
+    return job;
+  }
+
+  /**
+   * Crawl a site starting from `url`, BFS to `depth`.
+   *
+   * Returns immediately with `status="queued"`. Use `getJob` to poll, or
+   * `waitForJob` to block until done — both handle crawl and bulk jobIds
+   * transparently.
+   *
+   * Plan caps:
+   *   - Free        → `PermissionDeniedError` (`plan_not_supported`)
+   *   - Pro         → max depth 5, up to 1,000 pages per crawl
+   *   - Enterprise  → unlimited depth and pages
+   *
+   * Throws:
+   *   - `PermissionDeniedError`     — `plan_not_supported` (Free tier).
+   *   - `UnprocessableEntityError`  — `crawl_depth_exceeded`.
+   */
+  async crawl(url: string, options: CrawlOptions = {}): Promise<CrawlJob> {
+    const depth = options.depth ?? 1;
+    if (depth < 0) {
+      throw new Error("depth must be >= 0.");
+    }
+    const body = await this.request("POST", "/crawl", {
+      url,
+      depth,
+      render_js: options.renderJs === true,
+    });
+    return crawlJobFromResponse(body as Record<string, unknown>);
+  }
+
+  // ── Custom headers ─────────────────────────────────────────────────────────
+
+  /**
+   * Add or replace a per-request header for the rest of this client's life.
+   *
+   * Authorization / Content-Type / Accept are reserved — calls that try
+   * to set those are silently ignored. To rotate the bearer token, use
+   * `rotateKey()`.
+   */
+  setHeader(name: string, value: string): void {
+    if (RESERVED_HEADERS.has(name.toLowerCase())) return;
+    this.extraHeaders[name] = value;
+  }
+
+  /** Remove a header previously added via `headers:` or `setHeader()`. */
+  removeHeader(name: string): void {
+    delete this.extraHeaders[name];
+  }
+
+  /**
+   * Return your usage for the current billing period.
+   *
+   * Does not count toward your monthly quota.
+   */
+  async getUsage(): Promise<Usage> {
+    const body = await this.request("GET", "/usage");
+    return usageFromResponse(body as Record<string, unknown>);
+  }
+
+  /**
+   * Mint a new API key. The current key is invalidated immediately.
+   *
+   * The new raw key is in the returned `apiKey` field — store it before
+   * discarding the result. There is no recovery flow.
+   *
+   * The client auto-swaps to the new key for subsequent requests.
+   *
+   * Does not count toward your monthly quota.
+   */
+  async rotateKey(): Promise<RotatedKey> {
+    const body = await this.request("POST", "/keys/rotate");
+    const rotated = rotatedKeyFromResponse(body as Record<string, unknown>);
+    if (rotated.apiKey) {
+      this.apiKey = rotated.apiKey;
+    }
+    return rotated;
+  }
+
+  /**
+   * Internal: read the current API key. Exposed for tests.
+   * Not part of the public, semver-stable surface.
+   */
+  _getApiKey(): string {
+    return this.apiKey;
+  }
+
+  // ── Transport ──────────────────────────────────────────────────────────────
+
+  private async request(
+    method: string,
+    path: string,
+    json?: unknown,
+  ): Promise<unknown> {
+    const url = `${this.baseUrl}${path}`;
+    const headers = mergeHeaders(this.apiKey, this.extraHeaders);
+
+    const init: RequestInitWithSignal = { method, headers };
+    if (json !== undefined) {
+      init.body = JSON.stringify(json);
+    }
+
+    let controller: AbortController | null = null;
+    let timer: ReturnType<typeof setTimeout> | null = null;
+    if (this.timeoutMs > 0 && typeof AbortController !== "undefined") {
+      controller = new AbortController();
+      init.signal = controller.signal;
+      timer = setTimeout(() => controller!.abort(), this.timeoutMs);
+    }
+
+    let response: Response;
+    try {
+      response = await this.fetchImpl(url, init as RequestInit);
+    } catch (err) {
+      throw new APIConnectionError(
+        `Could not reach the WellMarked API: ${stringifyError(err)}`,
+        { cause: err },
+      );
+    } finally {
+      if (timer !== null) clearTimeout(timer);
+    }
+
+    let bodyText = "";
+    try {
+      bodyText = await response.text();
+    } catch (err) {
+      throw new APIConnectionError(
+        `Could not read API response body: ${stringifyError(err)}`,
+        { cause: err },
+      );
+    }
+
+    let body: unknown = null;
+    if (bodyText.length > 0) {
+      try {
+        body = JSON.parse(bodyText);
+      } catch {
+        body = null;
+      }
+    }
+
+    return parseResponse(response.status, body);
+  }
+}
+
+function parseResponse(statusCode: number, body: unknown): unknown {
+  let requestId: string | undefined;
+  if (body && typeof body === "object") {
+    const rid = (body as { request_id?: unknown }).request_id;
+    if (typeof rid === "string") requestId = rid;
+  }
+
+  if (statusCode >= 200 && statusCode < 300) {
+    if (body === null) {
+      // The API contract says every documented endpoint returns a JSON
+      // body on 2xx. A null body means the server broke that contract
+      // (or a middlebox stripped it); fail loudly rather than letting
+      // downstream parsing crash on `body.foo` of null.
+      throw new WellMarkedError(
+        `API returned HTTP ${statusCode} with no JSON body. ` +
+          "This is a contract violation — please report it.",
+        { statusCode },
+      );
+    }
+    return body;
+  }
+
+  throw fromResponse(statusCode, body, requestId);
+}
+
+function stringifyError(err: unknown): string {
+  if (err instanceof Error) {
+    return `${err.name}: ${err.message}`;
+  }
+  return String(err);
+}
+
+// Re-export the APIStatusError type so consumers can narrow without
+// pulling from "./errors" directly.
+export { APIStatusError };

package/src/errors.ts

@@ -0,0 +1,162 @@
+/**
+ * Exception hierarchy for the WellMarked SDK.
+ *
+ * Every HTTP error returned by the API is translated into a typed error
+ * whose class corresponds to the HTTP status and whose `code` matches the
+ * `error.code` field in the response body. Catch `WellMarkedError` for
+ * anything raised by the SDK; catch a more specific subclass when you want
+ * to handle one failure mode.
+ */
+
+export interface WellMarkedErrorOptions {
+  code?: string | undefined;
+  statusCode?: number | undefined;
+  retryAfter?: number | undefined;
+  requestId?: string | undefined;
+  cause?: unknown;
+}
+
+export class WellMarkedError extends Error {
+  readonly code: string | undefined;
+  readonly statusCode: number | undefined;
+  readonly retryAfter: number | undefined;
+  readonly requestId: string | undefined;
+
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message);
+    this.name = "WellMarkedError";
+    this.code = options.code;
+    this.statusCode = options.statusCode;
+    this.retryAfter = options.retryAfter;
+    this.requestId = options.requestId;
+    if (options.cause !== undefined) {
+      (this as { cause?: unknown }).cause = options.cause;
+    }
+    // Maintain proper prototype chain when transpiled to ES5.
+    Object.setPrototypeOf(this, new.target.prototype);
+  }
+}
+
+/** Raised when the SDK couldn't reach the API (DNS, TCP, TLS, timeout). */
+export class APIConnectionError extends WellMarkedError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "APIConnectionError";
+    Object.setPrototypeOf(this, APIConnectionError.prototype);
+  }
+}
+
+/** Raised for any non-2xx response from the API. */
+export class APIStatusError extends WellMarkedError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "APIStatusError";
+    Object.setPrototypeOf(this, APIStatusError.prototype);
+  }
+}
+
+/** 401 — missing or invalid API key. */
+export class AuthenticationError extends APIStatusError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "AuthenticationError";
+    Object.setPrototypeOf(this, AuthenticationError.prototype);
+  }
+}
+
+/** 403 — account inactive, plan does not allow this operation, or job belongs to another user. */
+export class PermissionDeniedError extends APIStatusError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "PermissionDeniedError";
+    Object.setPrototypeOf(this, PermissionDeniedError.prototype);
+  }
+}
+
+/** 404 — job not found or expired past the 6-hour retention window. */
+export class NotFoundError extends APIStatusError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "NotFoundError";
+    Object.setPrototypeOf(this, NotFoundError.prototype);
+  }
+}
+
+/**
+ * 422 — request was syntactically valid but couldn't be fulfilled.
+ *
+ * Common `code` values:
+ *   - `no_content`            — could not identify main content on the page
+ *   - `target_timeout`        — the target URL timed out
+ *   - `js_rendering_disabled` — `renderJs=true` but the server has it off
+ *   - `bulk_cap_exceeded`     — more URLs than the plan allows per request
+ *   - `crawl_depth_exceeded`  — requested depth above the plan cap
+ */
+export class UnprocessableEntityError extends APIStatusError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "UnprocessableEntityError";
+    Object.setPrototypeOf(this, UnprocessableEntityError.prototype);
+  }
+}
+
+/** 429 — monthly plan limit reached. `retryAfter` is the number of seconds until reset. */
+export class RateLimitError extends APIStatusError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "RateLimitError";
+    Object.setPrototypeOf(this, RateLimitError.prototype);
+  }
+}
+
+/** 5xx — something went wrong on the API side. */
+export class InternalServerError extends APIStatusError {
+  constructor(message: string, options: WellMarkedErrorOptions = {}) {
+    super(message, options);
+    this.name = "InternalServerError";
+    Object.setPrototypeOf(this, InternalServerError.prototype);
+  }
+}
+
+type APIStatusErrorCtor = new (
+  message: string,
+  options?: WellMarkedErrorOptions,
+) => APIStatusError;
+
+const STATUS_TO_EXC: Record<number, APIStatusErrorCtor> = {
+  401: AuthenticationError,
+  403: PermissionDeniedError,
+  404: NotFoundError,
+  422: UnprocessableEntityError,
+  429: RateLimitError,
+};
+
+/** Build the right error subclass for a given HTTP status + JSON body. */
+export function fromResponse(
+  statusCode: number,
+  body: unknown,
+  requestId?: string,
+): APIStatusError {
+  let code: string | undefined;
+  let message = `HTTP ${statusCode}`;
+  let retryAfter: number | undefined;
+
+  if (body && typeof body === "object" && "error" in body) {
+    const err = (body as { error?: unknown }).error;
+    if (err && typeof err === "object") {
+      const e = err as Record<string, unknown>;
+      if (typeof e.code === "string") code = e.code;
+      if (typeof e.message === "string") message = e.message;
+      if (typeof e.retry_after === "number") retryAfter = e.retry_after;
+    }
+  }
+
+  let Ctor: APIStatusErrorCtor;
+  if (statusCode >= 500) {
+    Ctor = InternalServerError;
+  } else {
+    Ctor = STATUS_TO_EXC[statusCode] ?? APIStatusError;
+  }
+
+  return new Ctor(message, { code, statusCode, retryAfter, requestId });
+}

package/src/index.ts

@@ -0,0 +1,45 @@
+/**
+ * Official JavaScript/TypeScript SDK for the WellMarked API.
+ *
+ *     import { WellMarked } from "wellmarked";
+ *
+ *     const wm = new WellMarked({ apiKey: "wm_..." });
+ *     const result = await wm.extract("https://example.com/article");
+ *     console.log(result.markdown);
+ *
+ * See https://wellmarked.io/docs for the full API reference.
+ */
+export { VERSION } from "./version.js";
+export {
+  WellMarked,
+  type WellMarkedOptions,
+  type ExtractOptions,
+  type BulkOptions,
+  type CrawlOptions,
+  type WaitForJobOptions,
+} from "./client.js";
+export {
+  type BulkItem,
+  type BulkJob,
+  type CrawlItem,
+  type CrawlJob,
+  type ExtractionMeta,
+  type ExtractResult,
+  type JobStatus,
+  type RotatedKey,
+  type TruncatedReason,
+  type Usage,
+  isBulkJob,
+  isCrawlJob,
+} from "./models.js";
+export {
+  APIConnectionError,
+  APIStatusError,
+  AuthenticationError,
+  InternalServerError,
+  NotFoundError,
+  PermissionDeniedError,
+  RateLimitError,
+  UnprocessableEntityError,
+  WellMarkedError,
+} from "./errors.js";