prostgles-server 4.2.483 → 4.2.484

package/lib/Filtering/Filtering.ts

@@ -0,0 +1,286 @@
+import type { FullFilter } from "prostgles-types";
+import {
+  BetweenFilterKeys,
+  CompareFilterKeys,
+  CompareInFilterKeys,
+  JsonbFilterKeys,
+  TextFilterKeys,
+  isDefined,
+  isEmpty,
+} from "prostgles-types";
+import { pgp } from "../DboBuilder/DboBuilderTypes";
+import { type SelectItemValidated } from "../DboBuilder/QueryBuilder/QueryBuilder";
+import { getFilterItemCondition } from "./getFilterItemCondition";
+
+export const FILTER_OPERANDS = [
+  ...TextFilterKeys,
+  ...JsonbFilterKeys,
+  ...CompareFilterKeys,
+  ...BetweenFilterKeys,
+  ...CompareInFilterKeys,
+] as const;
+
+export const FILTER_OPERAND_TO_SQL_OPERAND = Object.fromEntries(
+  FILTER_OPERANDS.map((filterOperand) => {
+    let sqlOperand = filterOperand as string;
+    if (filterOperand === "$eq") sqlOperand = "=";
+    else if (filterOperand === "$gt") sqlOperand = ">";
+    else if (filterOperand === "$gte") sqlOperand = ">=";
+    else if (filterOperand === "$lt") sqlOperand = "<";
+    else if (filterOperand === "$lte") sqlOperand = "<=";
+    else if (filterOperand === "$ne") sqlOperand = "<>";
+    else if (filterOperand === "$like") sqlOperand = "LIKE";
+    else if (filterOperand === "$ilike") sqlOperand = "ILIKE";
+    else if (filterOperand === "$nlike") sqlOperand = "NOT LIKE";
+    else if (filterOperand === "$nilike") sqlOperand = "NOT ILIKE";
+    else if (filterOperand === "$in") sqlOperand = "IN";
+    else if (filterOperand === "$nin") sqlOperand = "NOT IN";
+    else if (filterOperand === "$between") sqlOperand = "BETWEEN";
+    else if (filterOperand === "$notBetween") sqlOperand = "NOT BETWEEN";
+    else if (filterOperand === "$isDistinctFrom") sqlOperand = "IS DISTINCT FROM";
+    else if (filterOperand === "$isNotDistinctFrom") sqlOperand = "IS NOT DISTINCT FROM";
+    return [filterOperand, sqlOperand];
+  }),
+) as Record<(typeof FILTER_OPERANDS)[number], string>;
+
+/**
+ * Parse a single filter
+ * Ensure only single key objects reach this point
+ */
+type ParseFilterItemArgs = {
+  filter: FullFilter<void, void> | undefined;
+  select: SelectItemValidated[] | undefined;
+  tableAliasRaw: string | undefined;
+  allowedColumnNames: string[];
+};
+
+export const parseFilterItem = (
+  args: ParseFilterItemArgs,
+): { condition: string; columnsUsed: string[] } | undefined => {
+  const { filter: filterItem, select, tableAliasRaw, allowedColumnNames } = args;
+
+  if (!filterItem || isEmpty(filterItem)) return;
+
+  const makeError = (msg: string): never => {
+    throw `${msg}: ${JSON.stringify(filterItem, null, 2)}`;
+  };
+  const asValue = (v: any) => pgp.as.format("$1", [v]);
+
+  const filterEntries = Object.entries(filterItem);
+  const [firstFilterEntry, ...otherFilterEnties] = filterEntries;
+  if (!firstFilterEntry) {
+    return;
+
+    /**
+     * { field1: cond1, field2: cond2 }
+     */
+  } else if (otherFilterEnties.length) {
+    const items = filterEntries
+      .map(([filterKey, filterValue]) =>
+        parseFilterItem({
+          filter: { [filterKey]: filterValue },
+          select,
+          tableAliasRaw,
+          allowedColumnNames,
+        }),
+      )
+      .filter(isDefined);
+
+    const condition = items
+      .map((i) => i.condition)
+      .sort() /*  sorted to ensure duplicate subscription channels are not created due to different condition order */
+      .join(" AND ");
+    const columnsUsed = items.map((i) => i.columnsUsed).flat();
+
+    return { condition, columnsUsed };
+  }
+
+  // const fKey: string = filterKeys[0]!;
+  const [firstFilterKey, firstFilterValue] = firstFilterEntry;
+  let selItem: SelectItemValidated | undefined;
+  if (select) {
+    selItem = select.find((s) => firstFilterKey === s.alias);
+  }
+  let rightF = firstFilterValue;
+
+  const validateSelectedItemFilter = (selectedItem: SelectItemValidated | undefined) => {
+    const fields = selectedItem?.fields;
+    if (Array.isArray(fields) && fields.length) {
+      const dissallowedFields = fields.filter((fname) => !allowedColumnNames.includes(fname));
+      if (dissallowedFields.length) {
+        throw new Error(
+          `Invalid/disallowed columns found in filter: ${dissallowedFields.join(", ")}`,
+        );
+      }
+    }
+  };
+  const getLeftQ = (selItm: SelectItemValidated) => {
+    validateSelectedItemFilter(selItem);
+    if (selItm.type === "function" || selItm.type === "aggregation") return selItm.getQuery();
+    return selItm.getQuery(tableAliasRaw);
+  };
+
+  /**
+   * Parsed left side of the query
+   */
+  let leftQ: string | undefined; // = asName(selItem.alias);
+
+  /* 
+    Select item not found. 
+    Check if dot/json notation. Build obj if necessary 
+    */
+  const dot_notation_delims = ["->", "."];
+  if (!selItem) {
+    /* See if dot notation. Pick the best matching starting string */
+    if (select) {
+      selItem = select.find((s) =>
+        dot_notation_delims.find((delimiter) => firstFilterKey.startsWith(s.alias + delimiter)),
+      );
+      validateSelectedItemFilter(selItem);
+    }
+    if (!selItem) {
+      return makeError(
+        "Bad filter. Could not match to a column or alias or dot notation" +
+          select?.map((s) => s.alias).join(", "),
+      );
+    }
+
+    let remainingStr = firstFilterKey.slice(selItem.alias.length);
+
+    /* Is json path spec */
+    if (remainingStr.startsWith("->")) {
+      /** Has shorthand operand 'col->>key.<>'  */
+      const matchingOperand = CompareFilterKeys.find((operand) =>
+        remainingStr.endsWith(`.${operand}`),
+      );
+      if (matchingOperand) {
+        remainingStr = remainingStr.slice(0, -matchingOperand.length - 1);
+        rightF = { [matchingOperand]: rightF };
+      }
+
+      leftQ = getLeftQ(selItem);
+
+      /**
+       * get json path separators. Expecting -> to come first
+       */
+      type GetSepRes = { idx: number; sep: string } | undefined;
+      const getSep = (fromIdx = 0): GetSepRes => {
+        const strPart = remainingStr.slice(fromIdx);
+        let idx = strPart.indexOf("->");
+        const idxx = strPart.indexOf("->>");
+        if (idx > -1) {
+          /* if -> matches then check if it's the last separator */
+          if (idx === idxx) return { idx: idx + fromIdx, sep: "->>" };
+          return { idx: idx + fromIdx, sep: "->" };
+        }
+        idx = strPart.indexOf("->>");
+        if (idx > -1) {
+          return { idx: idx + fromIdx, sep: "->>" };
+        }
+
+        return undefined;
+      };
+
+      let currSep = getSep();
+      while (currSep) {
+        let nextSep = getSep(currSep.idx + currSep.sep.length);
+
+        let nextIdx = nextSep ? nextSep.idx : remainingStr.length;
+
+        /* If ending in set then add set as well into key */
+        if (nextSep && nextIdx + nextSep.sep.length === remainingStr.length) {
+          nextIdx = remainingStr.length;
+          nextSep = undefined;
+        }
+
+        leftQ +=
+          currSep.sep + asValue(remainingStr.slice(currSep.idx + currSep.sep.length, nextIdx));
+        currSep = nextSep;
+      }
+
+      /* 
+      Is collapsed filter spec  e.g. { "col.$ilike": 'text' } 
+      will transform into { col: { $ilike: ['text'] } }
+    */
+    } else if (remainingStr.startsWith(".")) {
+      leftQ = getLeftQ(selItem);
+
+      const getSep = (fromIdx = 0) => {
+        const idx = remainingStr.slice(fromIdx).indexOf(".");
+        if (idx > -1) return fromIdx + idx;
+        return idx;
+      };
+      let currIdx = getSep();
+      const res: any = {};
+      let curObj = res;
+
+      while (currIdx > -1) {
+        let nextIdx = getSep(currIdx + 1);
+        let nIdx = nextIdx > -1 ? nextIdx : remainingStr.length;
+
+        /* If ending in dot then add dot as well into key */
+        if (nextIdx + 1 === remainingStr.length) {
+          nIdx = remainingStr.length;
+          nextIdx = -1;
+        }
+
+        const key = remainingStr.slice(currIdx + 1, nIdx);
+        curObj[key] = nextIdx > -1 ? {} : firstFilterValue;
+        curObj = curObj[key];
+
+        currIdx = nextIdx;
+      }
+
+      rightF = res;
+    } else {
+      // console.trace(141, select, selItem, remainingStr)
+      makeError("Bad filter. Could not find the valid col name or alias or col json path");
+    }
+  } else {
+    leftQ = getLeftQ(selItem);
+  }
+
+  if (!leftQ) {
+    makeError("Internal error: leftQ missing?!");
+    return;
+  }
+
+  try {
+    const condition = getFilterItemCondition({
+      selItem,
+      leftQ,
+      rightF,
+    });
+    return {
+      condition,
+      columnsUsed: selItem.fields,
+    };
+  } catch (e) {
+    return makeError((e as Error).message);
+  }
+};
+
+// ensure pgp is not NULL!!!
+// const asValue = v => v;// pgp.as.value;
+
+// const filters: FilterSpec[] = [
+//   ...(["ilike", "like"].map(op => ({
+//     operands: ["$" + op],
+//     tsDataTypes: ["any"] as TSDataType[],
+//     tsDefinition: ` { $${op}: string } `,
+//     // data_types:
+//     getQuery: (leftQuery: string, rightVal: any) => {
+//       return `${leftQuery}::text ${op.toUpperCase()} ${asValue(rightVal)}::text`
+//     }
+//   }))),
+//   {
+//     operands: ["", "="],
+//     tsDataTypes: ["any"],
+//     tsDefinition: ` { "=": any } | any `,
+//     // data_types:
+//     getQuery: (leftQuery: string, rightVal: any) => {
+//       if(rightVal === null) return`${leftQuery} IS NULL `;
+//       return `${leftQuery} = ${asValue(rightVal)}`;
+//     }
+//   }
+// ];

package/lib/Filtering/getFilterItemCondition.ts

@@ -0,0 +1,205 @@
+import {
+  GeomFilterKeys,
+  GeomFilter_Funcs,
+  TextFilterKeys,
+  TextFilter_FullTextSearchFilterKeys,
+  includes,
+  isDefined,
+  isObject,
+} from "prostgles-types";
+import { parseFilterRightValue } from "./parseFilterRightValue";
+import { FILTER_OPERAND_TO_SQL_OPERAND, FILTER_OPERANDS } from "./Filtering";
+import type { SelectItemValidated } from "../DboBuilder/QueryBuilder/QueryBuilder";
+import { pgp } from "../DboBuilder/DboBuilderTypes";
+
+export const getFilterItemCondition = ({
+  selItem,
+  leftQ,
+  rightF,
+}: {
+  leftQ: string;
+  selItem: SelectItemValidated;
+  rightF: unknown;
+}): string => {
+  const asValue = (v: any) => pgp.as.format("$1", [v]);
+
+  const parseRightVal = (val: any, expect?: "csv" | "array" | "json" | "jsonb") => {
+    try {
+      return parseFilterRightValue(val, { selectItem: selItem, expect });
+    } catch (e: any) {
+      throw new Error(e);
+    }
+  };
+
+  /* Matching sel item */
+  if (isObject(rightF)) {
+    const filterKeys = Object.keys(rightF);
+    let filterOperand = filterKeys[0] as (typeof FILTER_OPERANDS)[number];
+
+    /** JSON cannot be compared so we'll cast it to TEXT */
+    if (selItem.column_udt_type === "json" || includes(TextFilterKeys, filterOperand)) {
+      leftQ += "::TEXT ";
+    }
+
+    /** It's an object key which means it's an equality comparison against a json object */
+    if (selItem.column_udt_type?.startsWith("json") && !includes(FILTER_OPERANDS, filterOperand)) {
+      return leftQ + " = " + parseRightVal(rightF);
+    }
+
+    let filterValue = rightF[filterOperand] as string | null | number | Date | any[];
+    const ALLOWED_FUNCS = [...GeomFilter_Funcs, ...TextFilter_FullTextSearchFilterKeys] as const;
+    let funcName: undefined | (typeof ALLOWED_FUNCS)[number];
+    let funcArgs: undefined | any[];
+
+    if (
+      selItem.column_udt_type === "interval" &&
+      isObject(rightF) &&
+      Object.values(rightF).every((v) => Number.isFinite(v))
+    ) {
+      filterOperand = "=";
+      filterValue = Object.entries(rightF)
+        .map(([k, v]) => `${v}${k}`)
+        .join(" ");
+    } else if (
+      (filterKeys.length !== 1 || !isDefined(filterOperand)) &&
+      selItem.column_udt_type !== "jsonb"
+    ) {
+      throw new Error("Bad filter. Expecting one key only");
+    } else if (isObject(filterValue) && !(filterValue instanceof Date)) {
+      /**
+       * Filter notation
+       * geom && st_makeenvelope(funcArgs)
+       */
+      const filterValueKeys = Object.keys(filterValue);
+      funcName = filterValueKeys[0] as any;
+      if (includes(ALLOWED_FUNCS, funcName)) {
+        funcArgs = filterValue[funcName as any];
+      } else {
+        funcName = undefined;
+      }
+    }
+
+    /** st_makeenvelope */
+    if (
+      includes(GeomFilterKeys, filterOperand) &&
+      funcName &&
+      includes(GeomFilter_Funcs, funcName)
+    ) {
+      /**
+       * If leftQ is geography then:
+       * - err can happen: 'Antipodal (180 degrees long) edge detected!'
+       * - inacurrate results at large envelopes due to the curvature of the earth
+       * https://gis.stackexchange.com/questions/78816/maximum-size-on-the-bounding-box-with-st-makeenvelope-and-and-geography-colum
+       */
+      if (funcName.toLowerCase() === "st_makeenvelope") {
+        leftQ += "::geometry";
+      }
+
+      return `${leftQ} ${filterOperand} ${funcName}${parseRightVal(funcArgs, "csv")}`;
+    } else if (["=", "$eq"].includes(filterOperand) && !funcName) {
+      if (filterValue === null) return leftQ + " IS NULL ";
+      return leftQ + " = " + parseRightVal(filterValue);
+    } else if (["<>", "$ne"].includes(filterOperand)) {
+      if (filterValue === null) return leftQ + " IS NOT NULL ";
+      return leftQ + " <> " + parseRightVal(filterValue);
+    } else if ([">", "$gt"].includes(filterOperand)) {
+      return leftQ + " > " + parseRightVal(filterValue);
+    } else if (["<", "$lt"].includes(filterOperand)) {
+      return leftQ + " < " + parseRightVal(filterValue);
+    } else if ([">=", "$gte"].includes(filterOperand)) {
+      return leftQ + " >=  " + parseRightVal(filterValue);
+    } else if (["<=", "$lte"].includes(filterOperand)) {
+      return leftQ + " <= " + parseRightVal(filterValue);
+    } else if (["$in", "$nin"].includes(filterOperand)) {
+      const isIn = filterOperand === "$in";
+      if (filterValue !== null && !Array.isArray(filterValue)) {
+        throw new Error("In filter expects an array");
+      }
+      if (!filterValue?.length) {
+        return isIn ? " FALSE " : " TRUE ";
+      }
+
+      const nonNullFilterValues = filterValue.filter((v) => v !== null);
+      const conditions: string[] = [];
+      if (nonNullFilterValues.length) {
+        conditions.push(
+          leftQ + (isIn ? " IN " : " NOT IN ") + parseRightVal(nonNullFilterValues, "csv"),
+        );
+      }
+      if (filterValue.includes(null)) {
+        const comparator = isIn ? " IS NULL " : " IS NOT NULL ";
+        conditions.push(` ${leftQ} ${comparator} `);
+      }
+      const joinedConditions = conditions.join(isIn ? " OR " : " AND ");
+      return conditions.length > 1 ? `(${joinedConditions})` : joinedConditions;
+    } else if (["$between"].includes(filterOperand)) {
+      if (!Array.isArray(filterValue) || filterValue.length !== 2) {
+        throw new Error("Between filter expects an array of two values");
+      }
+      return leftQ + " BETWEEN " + asValue(filterValue[0]) + " AND " + asValue(filterValue[1]);
+    } else if (["$ilike"].includes(filterOperand)) {
+      return leftQ + " ILIKE " + asValue(filterValue);
+    } else if (["$like"].includes(filterOperand)) {
+      return leftQ + " LIKE " + asValue(filterValue);
+    } else if (["$nilike"].includes(filterOperand)) {
+      return leftQ + " NOT ILIKE " + asValue(filterValue);
+    } else if (["$nlike"].includes(filterOperand)) {
+      return leftQ + " NOT LIKE " + asValue(filterValue);
+    } else if (filterOperand === "$isDistinctFrom" || filterOperand === "$isNotDistinctFrom") {
+      const operator = FILTER_OPERAND_TO_SQL_OPERAND[filterOperand];
+      return leftQ + ` ${operator} ` + asValue(filterValue);
+
+      /* MAYBE TEXT OR MAYBE ARRAY */
+    } else if (
+      ["@>", "<@", "$contains", "$containedBy", "$overlaps", "&&", "@@"].includes(filterOperand)
+    ) {
+      const operand =
+        filterOperand === "@@" ? "@@"
+        : ["@>", "$contains"].includes(filterOperand) ? "@>"
+        : ["&&", "$overlaps"].includes(filterOperand) ? "&&"
+        : "<@";
+
+      if (operand === "<@" || operand === "@>") {
+        if (selItem.column_udt_type === "jsonb") {
+          const filterValueAsString =
+            typeof filterValue === "string" ? filterValue : JSON.stringify(filterValue);
+          return leftQ + operand + parseRightVal(filterValueAsString);
+        }
+      }
+
+      /* Array for sure */
+      if (Array.isArray(filterValue)) {
+        return leftQ + operand + parseRightVal(filterValue, "array");
+
+        /* FTSQuery */
+      } else if (
+        ["@@"].includes(filterOperand) &&
+        includes(TextFilter_FullTextSearchFilterKeys, funcName)
+      ) {
+        let lq = `to_tsvector(${leftQ}::text)`;
+        if (selItem.columnPGDataType === "tsvector") lq = leftQ!;
+
+        const res = `${lq} ${operand} ` + `${funcName}${parseRightVal(funcArgs, "csv")}`;
+
+        return res;
+      } else {
+        throw new Error("Unrecognised filter operand: " + filterOperand + " ");
+      }
+    } else {
+      throw new Error("Unrecognised filter operand: " + filterOperand + " ");
+    }
+  }
+  /* Is an equal filter */
+  if (rightF === null) {
+    return leftQ + " IS NULL ";
+  }
+
+  /**
+   * Ensure that when comparing an array to a json column, the array is cast to json
+   */
+  let valueStr = asValue(rightF);
+  if (selItem.column_udt_type?.startsWith("json") && Array.isArray(rightF)) {
+    valueStr = pgp.as.format(`$1::jsonb`, [JSON.stringify(rightF)]);
+  }
+  return `${leftQ} = ${valueStr}`;
+};

package/lib/Filtering/parseFilterRightValue.ts

@@ -0,0 +1,24 @@
+import { pgp } from "../DboBuilder/DboBuilderTypes";
+import { type SelectItemValidated } from "../DboBuilder/QueryBuilder/QueryBuilder";
+
+type ParseRightValOpts = {
+  expect?: "csv" | "array" | "json" | "jsonb";
+  selectItem: SelectItemValidated | undefined;
+};
+export const parseFilterRightValue = (val: any, { expect, selectItem }: ParseRightValOpts) => {
+  const asValue = (v: any) => pgp.as.format("$1", [v]);
+  const checkIfArr = () => {
+    if (!Array.isArray(val)) {
+      throw "This type of filter/column expects an Array of items";
+    }
+  };
+  if (expect === "csv" || expect?.startsWith("json")) {
+    checkIfArr();
+    return pgp.as.format(`($1:${expect})`, [val]);
+  } else if (expect === "array" || selectItem?.columnPGDataType === "ARRAY") {
+    checkIfArr();
+    return pgp.as.format(" ARRAY[$1:csv]", [val]);
+  }
+
+  return asValue(val);
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "prostgles-server",
-  "version": "4.2.483",
+  "version": "4.2.484",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/dist/Filtering.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"Filtering.d.ts","sourceRoot":"","sources":["../lib/Filtering.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAkB,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAiBlE,OAAO,EAAE,KAAK,mBAAmB,EAAE,MAAM,wCAAwC,CAAC;AAElF,eAAO,MAAM,eAAe,oSAMlB,CAAC;AAEX,eAAO,MAAM,6BAA6B,EAqBrC,MAAM,CAAC,CAAC,OAAO,eAAe,EAAE,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;AAEtD;;;GAGG;AACH,KAAK,mBAAmB,GAAG;IACzB,MAAM,EAAE,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,SAAS,CAAC;IAC3C,MAAM,EAAE,mBAAmB,EAAE,GAAG,SAAS,CAAC;IAC1C,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,kBAAkB,EAAE,MAAM,EAAE,CAAC;CAC9B,CAAC;AAEF,eAAO,MAAM,eAAe,GAAI,MAAM,mBAAmB,KAAG,MAuW3D,CAAC;AAEF,KAAK,iBAAiB,GAAG;IACvB,MAAM,CAAC,EAAE,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,OAAO,CAAC;IAC5C,UAAU,EAAE,mBAAmB,GAAG,SAAS,CAAC;CAC7C,CAAC;AACF,eAAO,MAAM,qBAAqB,GAAI,KAAK,GAAG,EAAE,wBAAwB,iBAAiB,WAgBxF,CAAC"}