teraslice 3.0.0 → 3.1.0

package/dist/src/lib/cluster/services/api.js

@@ -6,7 +6,7 @@ import { RecoveryCleanupType } from '@terascope/job-components';
 import { parseErrorInfo, parseList, logError, TSError, startsWith, pWhile, isKey } from '@terascope/core-utils';
 import { ExecutionStatusEnum } from '@terascope/types';
 import { makeLogger } from '../../workers/helpers/terafoundation.js';
-import { makeTable, sendError, handleTerasliceRequest, getSearchOptions, createJobActiveQuery, addDeletedToQuery } from '../../utils/api_utils.js';
+import { makeTable, sendError, handleTerasliceRequest, getSearchOptions, createJobActiveQuery, addDeletedToQuery, addFilterToQuery } from '../../utils/api_utils.js';
 import { getPackageJSON } from '../../utils/file_utils.js';
 const terasliceVersion = getPackageJSON().version;
 function validateCleanupType(cleanupType) {
@@ -221,12 +221,13 @@ export class ApiService {
         });
         v1routes.get('/jobs', (req, res) => {
             const { active = '', deleted = 'false', ex } = req.query;
-            const { size, from, sort } = getSearchOptions(req);
+            const { size, from, sort, filter } = getSearchOptions(req);
             const requestHandler = handleTerasliceRequest(req, res, 'Could not retrieve list of jobs');
             requestHandler(() => {
                 validateGetDeletedOption(deleted);
-                const partialQuery = createJobActiveQuery(active);
-                const query = addDeletedToQuery(deleted, partialQuery);
+                let partialQuery = createJobActiveQuery(active);
+                partialQuery = addDeletedToQuery(deleted, partialQuery);
+                const query = addFilterToQuery(partialQuery, filter);
                 return typeof ex === 'string'
                     ? this.jobsService.getJobsWithExInfo(query, from, size, sort, ex.split(','))
                     : this.jobsStorage.search(query, from, size, sort);
@@ -356,21 +357,20 @@ export class ApiService {
         });
         v1routes.get([
             '/jobs/:jobId/errors',
-            '/jobs/:jobId/errors/:exId',
             '/ex/:exId/errors',
             '/ex/errors',
         ], (req, res) => {
-            const { size, from, sort } = getSearchOptions(req);
+            const { size, from, sort, filter } = getSearchOptions(req);
             const requestHandler = handleTerasliceRequest(req, res, 'Could not get errors for job');
             requestHandler(async () => {
                 const exId = await this._getExIdFromRequest(req, true);
-                const query = `state:error AND ex_id:"${exId}"`;
+                const query = addFilterToQuery(`state:error AND ex_id:"${exId}"`, filter);
                 return this.stateStorage.search(query, from, size, sort);
             });
         });
         v1routes.get('/ex', (req, res) => {
             const { status = '', deleted = 'false' } = req.query;
-            const { size, from, sort } = getSearchOptions(req);
+            const { size, from, sort, filter } = getSearchOptions(req);
             const requestHandler = handleTerasliceRequest(req, res, 'Could not retrieve list of execution contexts');
             requestHandler(async () => {
                 validateGetDeletedOption(deleted);
@@ -380,7 +380,8 @@ export class ApiService {
                     const statusTerms = statuses.map((s) => `_status:"${s}"`).join(' OR ');
                     partialQuery += ` AND (${statusTerms})`;
                 }
-                const query = addDeletedToQuery(deleted, partialQuery);
+                partialQuery = addDeletedToQuery(deleted, partialQuery);
+                const query = addFilterToQuery(partialQuery, filter);
                 return this.executionStorage.search(query, from, size, sort);
             });
         });
@@ -439,7 +440,7 @@ export class ApiService {
         });
         this.app.get('/txt/jobs', (req, res) => {
             const { active = '', deleted = 'false' } = req.query;
-            const { size, from, sort } = getSearchOptions(req);
+            const { size, from, sort, filter } = getSearchOptions(req);
             const defaults = ['job_id', 'name', 'active', 'lifecycle', 'slicers', 'workers', '_created', '_updated'];
             const requestHandler = handleTerasliceRequest(req, res, 'Could not get all jobs');
             requestHandler(async () => {
@@ -447,15 +448,16 @@ export class ApiService {
                 if (deleted !== 'false') {
                     defaults.push('_deleted_on');
                 }
-                const partialQuery = createJobActiveQuery(active);
-                const query = addDeletedToQuery(deleted, partialQuery);
+                let partialQuery = createJobActiveQuery(active);
+                partialQuery = addDeletedToQuery(deleted, partialQuery);
+                const query = addFilterToQuery(partialQuery, filter);
                 const jobs = await this.jobsStorage.search(query, from, size, sort);
                 return makeTable(req, defaults, jobs);
             });
         });
         this.app.get('/txt/ex', (req, res) => {
             const { deleted = 'false' } = req.query;
-            const { size, from, sort } = getSearchOptions(req);
+            const { size, from, sort, filter } = getSearchOptions(req);
             const defaults = ['name', 'lifecycle', 'slicers', 'workers', '_status', 'ex_id', 'job_id', '_created', '_updated'];
             const requestHandler = handleTerasliceRequest(req, res, 'Could not get all executions');
             requestHandler(async () => {
@@ -463,8 +465,9 @@ export class ApiService {
                 if (deleted !== 'false') {
                     defaults.push('_deleted_on');
                 }
-                const partialQuery = 'ex_id:*';
-                const query = addDeletedToQuery(deleted, partialQuery);
+                let partialQuery = 'ex_id:*';
+                partialQuery = addDeletedToQuery(deleted, partialQuery);
+                const query = addFilterToQuery(partialQuery, filter);
                 const exs = await this.executionStorage.search(query, from, size, sort);
                 return makeTable(req, defaults, exs);
             });

package/dist/src/lib/utils/api_utils.js

@@ -90,7 +90,8 @@ export function getSearchOptions(req, defaultSort = '_updated:desc') {
     const sort = req.query.sort || defaultSort;
     const size = parseQueryInt(req, 'size', 100);
     const from = parseQueryInt(req, 'from', 0);
-    return { size, from, sort };
+    const filter = req.query.filter || '';
+    return { size, from, sort, filter };
 }
 export function logTerasliceRequest(req) {
     const queryInfo = Object.entries(req.query)
@@ -114,4 +115,14 @@ export function addDeletedToQuery(deleted, query) {
     }
     return `${query} AND _deleted:true`;
 }
+/**
+ * Combines a base query from an endpoint with an optional filter using AND from lucene.
+ * @param query - The base Lucene query string
+ * @param filter - Optional filter query to append. getSearchOptions() will return an empty string
+ * if no filter is present.
+ * @returns The combined query, or original query if filter is empty
+ */
+export function addFilterToQuery(query, filter) {
+    return filter ? `(${query}) AND (${filter})` : query;
+}
 //# sourceMappingURL=api_utils.js.map

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "teraslice",
   "displayName": "Teraslice",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Distributed computing platform for processing JSON data",
   "homepage": "https://github.com/terascope/teraslice#readme",
   "bugs": {
@@ -62,7 +62,8 @@
     "semver": "~7.7.2",
     "socket.io": "~4.8.1",
     "socket.io-client": "~4.8.1",
-    "terafoundation": "~2.0.0",
+    "terafoundation": "~2.0.1",
+    "terafoundation_kafka_connector": "~2.0.0",
     "uuid": "~13.0.0"
   },
   "devDependencies": {