ydb-qdrant 8.1.0 → 9.0.3

package/dist/logging/DeployLogFormatter.js

@@ -0,0 +1,131 @@
+import { Transform } from "node:stream";
+const RESERVED_FIELDS = new Set([
+    "msg",
+    "message",
+    "level",
+    "levelStr",
+    "@fields",
+    "stackTrace",
+    "stack",
+]);
+function isObject(value) {
+    return typeof value === "object" && value !== null;
+}
+function isPinoLine(value) {
+    if (!isObject(value)) {
+        return false;
+    }
+    return "hostname" in value && "pid" in value;
+}
+function mapPinoLevelToDeploy(level) {
+    if (typeof level !== "number") {
+        return undefined;
+    }
+    switch (level) {
+        case 60:
+        case 50:
+            return { level: 40000, levelStr: "ERROR" };
+        case 40:
+            return { level: 30000, levelStr: "WARNING" };
+        case 30:
+            return { level: 20000, levelStr: "INFO" };
+        case 20:
+        case 10:
+            return { level: 10000, levelStr: "DEBUG" };
+        default:
+            return undefined;
+    }
+}
+function getDeployMessage(obj) {
+    if (typeof obj.msg === "string") {
+        return obj.msg;
+    }
+    if (typeof obj.message === "string") {
+        return obj.message;
+    }
+    return "";
+}
+function chunkToUtf8Text(chunk) {
+    if (typeof chunk === "string") {
+        return chunk;
+    }
+    if (Buffer.isBuffer(chunk)) {
+        return chunk.toString("utf8");
+    }
+    if (chunk instanceof Uint8Array) {
+        return Buffer.from(chunk).toString("utf8");
+    }
+    return String(chunk);
+}
+function convertToDeployLine(obj) {
+    const fields = {};
+    const msg = getDeployMessage(obj);
+    const mapped = mapPinoLevelToDeploy(obj.level);
+    const log = {
+        "@fields": fields,
+        message: msg,
+        ...(mapped ? { level: mapped.level, levelStr: mapped.levelStr } : {}),
+    };
+    if (typeof obj.stack === "string" && obj.stack.length > 0) {
+        log.stackTrace = obj.stack;
+    }
+    for (const [key, value] of Object.entries(obj)) {
+        if (!RESERVED_FIELDS.has(key)) {
+            fields[key] = value;
+        }
+    }
+    return JSON.stringify(log);
+}
+export function createDeployLogFormatter() {
+    let buffered = "";
+    return new Transform({
+        readableObjectMode: false,
+        writableObjectMode: false,
+        transform(chunk, _encoding, callback) {
+            const text = chunkToUtf8Text(chunk);
+            buffered += text;
+            let nl;
+            while ((nl = buffered.indexOf("\n")) !== -1) {
+                const line = buffered.slice(0, nl);
+                buffered = buffered.slice(nl + 1);
+                if (line.trim().length === 0) {
+                    this.push("\n");
+                    continue;
+                }
+                try {
+                    const parsed = JSON.parse(line);
+                    if (!isPinoLine(parsed)) {
+                        this.push(line + "\n");
+                        continue;
+                    }
+                    this.push(convertToDeployLine(parsed) + "\n");
+                }
+                catch {
+                    this.push(line + "\n");
+                }
+            }
+            callback();
+        },
+        flush(callback) {
+            if (buffered.trim().length === 0) {
+                callback();
+                return;
+            }
+            const line = buffered;
+            buffered = "";
+            try {
+                const parsed = JSON.parse(line);
+                if (!isPinoLine(parsed)) {
+                    this.push(line + "\n");
+                }
+                else {
+                    this.push(convertToDeployLine(parsed) + "\n");
+                }
+            }
+            catch {
+                this.push(line + "\n");
+            }
+            callback();
+        },
+    });
+}

package/dist/logging/logger.js

@@ -1,3 +1,15 @@
 import pino from "pino";
 import { LOG_LEVEL } from "../config/env.js";
-export const logger = pino({ level: LOG_LEVEL });
+import { createDeployLogFormatter } from "./DeployLogFormatter.js";
+import { getRequestContextLogFields } from "./requestContext.js";
+const deployFormatter = createDeployLogFormatter();
+deployFormatter.pipe(process.stdout);
+export const logger = pino({
+    level: LOG_LEVEL,
+    serializers: {
+        err: pino.stdSerializers.err,
+    },
+    mixin() {
+        return getRequestContextLogFields();
+    },
+}, deployFormatter);

package/dist/logging/requestContext.d.ts

@@ -0,0 +1,17 @@
+export type RequestContext = {
+    requestId: string;
+    method: string;
+    url: string;
+    traceparent?: string;
+    amznTraceId?: string;
+    requestStartNs?: bigint;
+};
+export declare function runWithRequestContext<T>(context: RequestContext, fn: () => T): T;
+export declare function getRequestContext(): RequestContext | undefined;
+export declare function getRequestContextLogFields(): Partial<RequestContext> | Record<string, never>;
+export declare function getMonotonicTimeNs(): bigint;
+export declare function elapsedMsBetween(startNs: bigint, endNs: bigint): number;
+export declare function elapsedMsSince(startNs: bigint): number;
+export declare function getRequestStartNs(): bigint | undefined;
+export declare function elapsedMsFromRequestStart(markNs: bigint | undefined): number | undefined;
+export declare function getElapsedMsSinceRequestStart(): number | undefined;

package/dist/logging/requestContext.js

@@ -0,0 +1,43 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+const requestContextStorage = new AsyncLocalStorage();
+export function runWithRequestContext(context, fn) {
+    return requestContextStorage.run(context, fn);
+}
+export function getRequestContext() {
+    return requestContextStorage.getStore();
+}
+export function getRequestContextLogFields() {
+    const context = getRequestContext();
+    if (!context) {
+        return {};
+    }
+    const { requestStartNs: _requestStartNs, ...publicFields } = context;
+    void _requestStartNs;
+    return publicFields;
+}
+export function getMonotonicTimeNs() {
+    return process.hrtime.bigint();
+}
+export function elapsedMsBetween(startNs, endNs) {
+    return Number((endNs - startNs) / 1000000n);
+}
+export function elapsedMsSince(startNs) {
+    return elapsedMsBetween(startNs, getMonotonicTimeNs());
+}
+export function getRequestStartNs() {
+    return getRequestContext()?.requestStartNs;
+}
+export function elapsedMsFromRequestStart(markNs) {
+    const requestStartNs = getRequestStartNs();
+    if (requestStartNs === undefined || markNs === undefined) {
+        return undefined;
+    }
+    return elapsedMsBetween(requestStartNs, markNs);
+}
+export function getElapsedMsSinceRequestStart() {
+    const requestStartNs = getRequestStartNs();
+    if (requestStartNs === undefined) {
+        return undefined;
+    }
+    return elapsedMsSince(requestStartNs);
+}

package/dist/middleware/requestLogger.js

@@ -1,14 +1,142 @@
+import { randomUUID } from "node:crypto";
 import { logger } from "../logging/logger.js";
+import { getUpsertRequestTimeoutPhase, getUpsertRequestTimeoutMs, isUpsertRequestTimedOut, } from "./upsertRequestTimeout.js";
+import { getUpsertBodyPhaseTimeoutLogFields } from "./upsertBodyPhase.js";
+import { runWithRequestContext, getMonotonicTimeNs, } from "../logging/requestContext.js";
+const MAX_REQUEST_ID_LENGTH = 128;
+const SAFE_REQUEST_ID_RE = /^[A-Za-z0-9._:-]+$/;
+function normalizeRequestId(value) {
+    const trimmed = value?.trim();
+    if (!trimmed) {
+        return undefined;
+    }
+    if (trimmed.length > MAX_REQUEST_ID_LENGTH) {
+        return undefined;
+    }
+    return SAFE_REQUEST_ID_RE.test(trimmed) ? trimmed : undefined;
+}
+function parseClientIpFromXForwardedFor(value) {
+    if (!value) {
+        return undefined;
+    }
+    const first = value.split(",")[0]?.trim();
+    return first && first.length > 0 ? first : undefined;
+}
+function parseXForwardedForChainLen(value) {
+    if (!value) {
+        return 0;
+    }
+    return value
+        .split(",")
+        .map((s) => s.trim())
+        .filter((s) => s.length > 0).length;
+}
 export function requestLogger(req, res, next) {
     const start = Date.now();
-    const tenant = req.header("X-Tenant-Id") ?? "default";
+    const traceparent = req.header("traceparent") ?? undefined;
+    const amznTraceId = req.header("X-Amzn-Trace-Id") ?? undefined;
+    const requestIdFromHeader = normalizeRequestId(req.header("X-Request-Id") ?? undefined);
+    const requestId = requestIdFromHeader ?? randomUUID();
+    res.setHeader("X-Request-Id", requestId);
+    const userAgent = req.header("User-Agent") ?? undefined;
     const { method, originalUrl } = req;
+    const requestContext = {
+        requestId,
+        method,
+        url: originalUrl,
+        traceparent,
+        amznTraceId,
+        requestStartNs: getMonotonicTimeNs(),
+    };
     const contentLength = req.header("content-length");
+    const xForwardedFor = req.header("x-forwarded-for");
     const queryKeys = Object.keys(req.query || {});
-    logger.info({ method, url: originalUrl, tenant, contentLength, queryKeys }, "req");
-    res.on("finish", () => {
-        const ms = Date.now() - start;
-        logger.info({ method, url: originalUrl, tenant, status: res.statusCode, ms }, "res");
+    let finished = false;
+    let aborted = false;
+    let closedLogged = false;
+    runWithRequestContext(requestContext, () => {
+        logger.info({
+            userAgent,
+            contentLength,
+            clientIp: parseClientIpFromXForwardedFor(xForwardedFor) ??
+                req.socket.remoteAddress ??
+                undefined,
+            ipChainLen: parseXForwardedForChainLen(xForwardedFor),
+            hasXForwardedFor: xForwardedFor !== undefined,
+            queryKeys,
+        }, "req");
+        req.on("aborted", () => {
+            aborted = true;
+        });
+        res.on("finish", () => {
+            runWithRequestContext(requestContext, () => {
+                finished = true;
+                const ms = Date.now() - start;
+                const timeoutMs = getUpsertRequestTimeoutMs(req);
+                const timeoutPhase = getUpsertRequestTimeoutPhase(req);
+                if (isUpsertRequestTimedOut(req) && timeoutMs !== undefined) {
+                    logger.warn({
+                        userAgent,
+                        contentLength,
+                        status: res.statusCode,
+                        ms,
+                        bytesRead: req.socket.bytesRead,
+                        timeoutMs,
+                        timeoutPhase,
+                        ...getUpsertBodyPhaseTimeoutLogFields(req),
+                    }, "req timed out");
+                    return;
+                }
+                const payload = {
+                    userAgent,
+                    status: res.statusCode,
+                    ms,
+                };
+                if (res.statusCode >= 500) {
+                    logger.error(payload, "res");
+                    return;
+                }
+                if (res.statusCode >= 300) {
+                    logger.warn(payload, "res");
+                    return;
+                }
+                logger.info(payload, "res");
+            });
+        });
+        res.on("close", () => {
+            runWithRequestContext(requestContext, () => {
+                if (finished || closedLogged) {
+                    return;
+                }
+                closedLogged = true;
+                const ms = Date.now() - start;
+                const bytesRead = req.socket.bytesRead;
+                const payload = {
+                    userAgent,
+                    contentLength,
+                    status: res.statusCode,
+                    ms,
+                    bytesRead,
+                    aborted,
+                };
+                const timeoutMs = getUpsertRequestTimeoutMs(req);
+                const timeoutPhase = getUpsertRequestTimeoutPhase(req);
+                if (isUpsertRequestTimedOut(req) && timeoutMs !== undefined) {
+                    logger.warn({
+                        ...payload,
+                        timeoutMs,
+                        timeoutPhase,
+                        ...getUpsertBodyPhaseTimeoutLogFields(req),
+                    }, "req timed out");
+                    return;
+                }
+                if (aborted) {
+                    logger.warn(payload, "req aborted");
+                    return;
+                }
+                logger.warn(payload, "req closed before response finished");
+            });
+        });
+        next();
     });
-    next();
 }

package/dist/middleware/upsertBodyPhase.d.ts

@@ -0,0 +1,6 @@
+import type { NextFunction, Request, Response } from "express";
+export declare function instrumentUpsertRequestBody(req: Request, _res: Response, next: NextFunction): void;
+export declare function verifyUpsertRequestBody(req: Request, _res: Response, buf: Buffer): void;
+export declare function logUpsertBodyPhase(req: Request, res: Response, next: NextFunction): void;
+export declare function logUpsertBodyPhaseError(err: unknown, req: Request, _res: Response, next: NextFunction): void;
+export declare function getUpsertBodyPhaseTimeoutLogFields(req: Request): Record<string, unknown>;

package/dist/middleware/upsertBodyPhase.js

@@ -0,0 +1,184 @@
+import { UPSERT_BODY_TIMEOUT_MS } from "../config/env.js";
+import { logger } from "../logging/logger.js";
+import { elapsedMsBetween, elapsedMsFromRequestStart, getRequestStartNs, getMonotonicTimeNs, } from "../logging/requestContext.js";
+import { isUpsertRequestTarget, isUpsertRequestTimedOut, markUpsertBodyPhaseCompleted, respondUpsertRequestTimedOut, } from "./upsertRequestTimeout.js";
+const UPSERT_BODY_PHASE_STATE = Symbol("upsertBodyPhaseState");
+function asUpsertBodyAwareRequest(req) {
+    return req;
+}
+function getUpsertBodyPhaseState(req) {
+    return asUpsertBodyAwareRequest(req)[UPSERT_BODY_PHASE_STATE];
+}
+function getOrCreateUpsertBodyPhaseState(req) {
+    const request = asUpsertBodyAwareRequest(req);
+    request[UPSERT_BODY_PHASE_STATE] ??= {
+        observedBodyBytes: 0,
+        bodyParseCompleted: false,
+        bodyPhaseLogged: false,
+    };
+    return request[UPSERT_BODY_PHASE_STATE];
+}
+function appendObservedBodyBytes(state, chunk) {
+    if (typeof chunk === "string") {
+        state.observedBodyBytes += Buffer.byteLength(chunk);
+        return;
+    }
+    if (chunk instanceof Uint8Array) {
+        state.observedBodyBytes += chunk.byteLength;
+    }
+}
+function parseContentLength(req) {
+    const raw = req.header("content-length");
+    if (!raw) {
+        return undefined;
+    }
+    const parsed = Number(raw);
+    return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
+}
+function buildUpsertBodyPhaseLogFields(req, state) {
+    const timeToBodyParsedMs = elapsedMsFromRequestStart(state.bodyParsedAtNs);
+    const timeToBodyBufferedMs = elapsedMsFromRequestStart(state.bodyBufferedAtNs);
+    const timeToFirstChunkMs = elapsedMsFromRequestStart(state.firstChunkAtNs);
+    const bodyReadMs = state.firstChunkAtNs !== undefined && state.bodyBufferedAtNs !== undefined
+        ? elapsedMsBetween(state.firstChunkAtNs, state.bodyBufferedAtNs)
+        : undefined;
+    const jsonParseMs = state.bodyBufferedAtNs !== undefined && state.bodyParsedAtNs !== undefined
+        ? elapsedMsBetween(state.bodyBufferedAtNs, state.bodyParsedAtNs)
+        : undefined;
+    return {
+        phase: "upsertBody",
+        timeToFirstChunkMs,
+        timeToBodyBufferedMs,
+        timeToBodyParsedMs,
+        bodyReadMs,
+        jsonParseMs,
+        observedBodyBytes: state.observedBodyBytes,
+        parsedBodyBytes: state.parsedBodyBytes,
+        bodyParseCompleted: state.bodyParseCompleted,
+        contentLength: parseContentLength(req),
+    };
+}
+function didBodyBudgetExpire(state) {
+    if (UPSERT_BODY_TIMEOUT_MS <= 0 || state.bodyParsedAtNs === undefined) {
+        return false;
+    }
+    const requestStartNs = getRequestStartNs();
+    if (requestStartNs === undefined) {
+        return false;
+    }
+    return (state.bodyParsedAtNs - requestStartNs >=
+        BigInt(UPSERT_BODY_TIMEOUT_MS) * 1000000n);
+}
+function extractBodyErrorType(err) {
+    if (!err || typeof err !== "object") {
+        return undefined;
+    }
+    return "type" in err && typeof err.type === "string" ? err.type : undefined;
+}
+function extractBodyErrorStatus(err) {
+    if (!err || typeof err !== "object") {
+        return undefined;
+    }
+    if ("statusCode" in err && typeof err.statusCode === "number") {
+        return err.statusCode;
+    }
+    return "status" in err && typeof err.status === "number"
+        ? err.status
+        : undefined;
+}
+export function instrumentUpsertRequestBody(req, _res, next) {
+    if (!isUpsertRequestTarget(req)) {
+        next();
+        return;
+    }
+    const state = getOrCreateUpsertBodyPhaseState(req);
+    // This must stay immediately before express.json() and call next()
+    // synchronously: attaching a "data" listener switches the request stream
+    // into flowing mode before body-parser attaches its own listeners.
+    req.on("data", (chunk) => {
+        if (state.firstChunkAtNs === undefined) {
+            state.firstChunkAtNs = getMonotonicTimeNs();
+        }
+        appendObservedBodyBytes(state, chunk);
+    });
+    next();
+}
+export function verifyUpsertRequestBody(req, _res, buf) {
+    if (!isUpsertRequestTarget(req)) {
+        return;
+    }
+    const state = getOrCreateUpsertBodyPhaseState(req);
+    state.bodyBufferedAtNs = getMonotonicTimeNs();
+    state.parsedBodyBytes = buf.byteLength;
+    markUpsertBodyPhaseCompleted(req);
+}
+export function logUpsertBodyPhase(req, res, next) {
+    if (!isUpsertRequestTarget(req)) {
+        next();
+        return;
+    }
+    const state = getUpsertBodyPhaseState(req);
+    if (!state || state.bodyPhaseLogged) {
+        next();
+        return;
+    }
+    if (isUpsertRequestTimedOut(req)) {
+        // Timeout response already went out. Preserve parse timing for the
+        // timeout log path, but suppress the normal success log and stop here.
+        state.bodyParsedAtNs = getMonotonicTimeNs();
+        state.bodyParseCompleted = true;
+        markUpsertBodyPhaseCompleted(req);
+        return;
+    }
+    state.bodyParsedAtNs = getMonotonicTimeNs();
+    state.bodyParseCompleted = true;
+    if (didBodyBudgetExpire(state)) {
+        respondUpsertRequestTimedOut({
+            req,
+            res,
+            timeoutMs: UPSERT_BODY_TIMEOUT_MS,
+            timeoutPhase: "parse",
+        });
+        return;
+    }
+    state.bodyPhaseLogged = true;
+    logger.info(buildUpsertBodyPhaseLogFields(req, state), "upsert: body phase");
+    next();
+}
+export function logUpsertBodyPhaseError(err, req, _res, next) {
+    if (!isUpsertRequestTarget(req)) {
+        next(err);
+        return;
+    }
+    const state = getUpsertBodyPhaseState(req);
+    if (!state || state.bodyPhaseLogged) {
+        next(err);
+        return;
+    }
+    if (isUpsertRequestTimedOut(req)) {
+        markUpsertBodyPhaseCompleted(req);
+        next(err);
+        return;
+    }
+    state.bodyPhaseLogged = true;
+    markUpsertBodyPhaseCompleted(req);
+    logger.warn({
+        ...buildUpsertBodyPhaseLogFields(req, state),
+        bodyErrorType: extractBodyErrorType(err),
+        bodyErrorStatus: extractBodyErrorStatus(err),
+    }, "upsert: body phase failed");
+    next(err);
+}
+export function getUpsertBodyPhaseTimeoutLogFields(req) {
+    if (!isUpsertRequestTarget(req)) {
+        return {};
+    }
+    const state = getUpsertBodyPhaseState(req);
+    if (!state) {
+        return {};
+    }
+    return {
+        ...buildUpsertBodyPhaseLogFields(req, state),
+        bodyPhaseLogged: state.bodyPhaseLogged,
+    };
+}

package/dist/middleware/upsertRequestTimeout.d.ts

@@ -0,0 +1,16 @@
+import type { NextFunction, Request, Response } from "express";
+export type UpsertTimeoutPhase = "body" | "parse" | "processing";
+export declare function isUpsertRequestTarget(req: Request): boolean;
+export declare function markUpsertRequestTimedOut(req: Request, timeoutMs: number, timeoutPhase: UpsertTimeoutPhase): void;
+export declare function isUpsertRequestTimedOut(req: Request): boolean;
+export declare function getUpsertRequestTimeoutMs(req: Request): number | undefined;
+export declare function getUpsertRequestTimeoutPhase(req: Request): UpsertTimeoutPhase | undefined;
+export declare function respondUpsertRequestTimedOut(args: {
+    req: Request;
+    res: Response;
+    timeoutMs: number;
+    timeoutPhase: UpsertTimeoutPhase;
+}): boolean;
+export declare function markUpsertBodyPhaseCompleted(req: Request): void;
+export declare function upsertBodyTimeout(req: Request, res: Response, next: NextFunction): void;
+export declare function upsertProcessingTimeout(req: Request, res: Response, next: NextFunction): void;