scimgateway 6.1.3 → 6.1.4

package/README.md

@@ -18,7 +18,7 @@
 Latest news:  
 
 - Bun binary build is now supported, allowing SCIM Gateway to be compiled into a single executable binary for simplified deployment and execution. SCIM Gateway can now run as an ES module (TypeScript) in Node.js.
-- Major release **v6.0.0** introduces changes to API method response bodies (not SCIM-related) and a new method `publicApi()` for handling public path `/pub/api` requests with no authentication required. In addition, the configuration option `bearerJwtAzure.tenantIdGUID` has been replaced by `bearerJwt.azureTenantId`. See the version history for details.
+- Major release **v6.0.0** introduces changes to API method responses (not SCIM-related) and a new method `publicApi()` for handling public path `/pub/api` requests with no authentication required. In addition, the configuration option `bearerJwtAzure.tenantIdGUID` has been replaced by `bearerJwt.azureTenantId`. See the version history for details.
 - Support for Entra ID [Federated Identity Credentials](https://learn.microsoft.com/en-us/graph/api/resources/federatedidentitycredentials-overview?view=graph-rest-1.0) has been added through internal JWKS (JSON Web Key Set), allowing SCIM Gateway to access Microsoft Entra–protected resources without the need to manage secrets
 - External JWKS (JSON Web Key Set) is now supported by JWT authentication, allowing external applications to access SCIM Gateway without the need to manage secrets
 - [Azure Relay](https://learn.microsoft.com/en-us/azure/azure-relay/relay-what-is-it) is now supported for secure and hassle-free outbound-only communication — with just one minute of configuration
@@ -1303,6 +1303,14 @@ MIT © [Jarle Elshaug](https://www.elshaug.xyz)
 
 ## Change log
 
+### v6.1.4
+
+[Fixed]
+
+- plugin-entra-id, OData paging was not working, so some users/groups/members might be missing
+- helper-rest, OData paging
+- user’s group membership did not iterate through paging and may be incomplete
+
 ### v6.1.3
 
 [Fixed]

package/config/plugin-entra-id.json

@@ -195,6 +195,10 @@
           "mapTo": "companyName",
           "type": "string"
         },
+        "employeeHireDate": {
+          "mapTo": "employeeHireDate",
+          "type": "string"
+        },
         "employeeOrgData.costCenter": {
           "mapTo": "employeeOrgData.costCenter",
           "type": "string"
@@ -239,6 +243,10 @@
           "type": "array",
           "typeInbound": "string"
         },
+        "faxNumber": {
+          "mapTo": "faxNumber",
+          "type": "string"
+        },
         "country": {
           "mapTo": "country",
           "type": "string"
@@ -298,7 +306,7 @@
           "type": "string"
         },
         "displayName": {
-          "mapTo": "displayName,externalId",
+          "mapTo": "displayName",
           "type": "string"
         },
         "securityEnabled": {

package/lib/helper-rest.ts

@@ -65,10 +65,9 @@ export class HelperRest {
    * getAccessToken returns oauth accesstoken object
    * @param baseEntity 
    * @param connectionObj endpoint.entity.baseEntity.connection
-   * @param ctx 
    * @returns { access_token: 'xxx', token_type: 'Bearer/Basic', validTo: 'xxx' }
    */
-  public async getAccessToken(baseEntity: string, connectionObj: Record<string, any>, ctx?: Record<string, any> | undefined) { // public in case token is needed for other logic e.g. sending mail
+  public async getAccessToken(baseEntity: string, connectionObj: Record<string, any>) { // public in case token is needed for other logic e.g. sending mail
     await this.lock.acquire()
     const d = Math.floor(Date.now() / 1000) // seconds (unix time)
     if (this._serviceClient[baseEntity]?.accessToken?.validTo >= d + 30) { // avoid simultaneously token requests
@@ -229,7 +228,7 @@ export class HelperRest {
               if (!this.scimgateway.jwk[kid]) {
                 this.scimgateway.jwk[kid] = { publicKey, privateKey }
                 const ttl = 5 * 60
-                ;(async () => {
+                  ; (async () => {
                   setTimeout(async () => {
                     delete this.scimgateway.jwk[kid]
                   }, ttl * 1000)
@@ -393,7 +392,7 @@ export class HelperRest {
       if (!connOpt.headers) connOpt.headers = {}
       connOpt.headers['Content-Type'] = 'application/x-www-form-urlencoded' // body must be query string formatted (no JSON)
 
-      const response = await this.doRequest(baseEntity, method, tokenUrl, form, ctx, connOpt)
+      const response = await this.doRequest(baseEntity, method, tokenUrl, form, undefined, connOpt)
       if (!response.body) {
         const err = new Error(`[${action}] No data retrieved from: ${method} ${tokenUrl}`)
         throw (err)
@@ -452,7 +451,7 @@ export class HelperRest {
           if (this._serviceClient[baseEntity].accessToken.validTo < d + 30) { // less than 30 sec before token expiration
             this.scimgateway.logDebug(baseEntity, `${action}: Accesstoken about to expire in ${this._serviceClient[baseEntity].accessToken.validTo - d} seconds`)
             try {
-              const accessToken = await this.getAccessToken(baseEntity, connectionObj, ctx)
+              const accessToken = await this.getAccessToken(baseEntity, connectionObj)
               this._serviceClient[baseEntity].accessToken = accessToken
               this._serviceClient[baseEntity].options.headers['Authorization'] = `${accessToken.token_type} ${accessToken.access_token}`
             } catch (err) {
@@ -512,7 +511,7 @@ export class HelperRest {
           }
         }
 
-        param.accessToken = await this.getAccessToken(baseEntity, connectionObj, ctx)
+        param.accessToken = await this.getAccessToken(baseEntity, connectionObj)
         if (param.accessToken?.access_token && param.accessToken?.token_type) {
           param.options.headers['Authorization'] = `${param.accessToken.token_type} ${param.accessToken.access_token}`
         } else { // no auth or PassTrough
@@ -561,8 +560,6 @@ export class HelperRest {
 
         // OData support
         this._serviceClient[baseEntity].nextLink = {} // OData pagination (Entra ID)
-        this._serviceClient[baseEntity].nextLink.users = null
-        this._serviceClient[baseEntity].nextLink.groups = null
       }
 
       if (ctx?.headers?.get) { // Auth PassThrough using ctx header
@@ -679,7 +676,35 @@ export class HelperRest {
           options.body = dataString
         } else if (options.headers) delete options.headers['Content-Type']
 
-        const url = `${options.protocol}//${options.host}${options.port ? ':' + options.port : ''}${options.path}`
+        let url = `${options.protocol}//${options.host}${options.port ? ':' + options.port : ''}${options.path}`
+        if (url.includes('$count=true') && url.includes('graph.microsoft.com')) {
+          options.headers['ConsistencyLevel'] = 'eventual'
+        }
+
+        if (this._serviceClient[baseEntity]?.nextLink[url]) {
+          if (ctx?.paging?.startIndex && ctx.paging.startIndex > 1) {
+            if (ctx.paging.startIndex === this._serviceClient[baseEntity]?.nextLink[url].startIndex) {
+              url = this._serviceClient[baseEntity]?.nextLink[url]['@odata.nextLink']
+            } else {
+              if (!ctx) ctx = {}
+              if (!ctx.paging) ctx.paging = {}
+              if (this._serviceClient[baseEntity]?.nextLink[url].totalResults
+                && ctx.paging.startIndex > this._serviceClient[baseEntity]?.nextLink[url].totalResults) {
+                ctx.paging.totalResults = this._serviceClient[baseEntity]?.nextLink[url].totalResults
+                return { body: { value: [] } }
+              } else {
+                // reset the paging cursor - none expected startIndex sequence, using default none paged url
+                ctx.paging.startIndex = 1 // caller should check and return this new startIndex in final response
+                delete this._serviceClient[baseEntity].nextLink[url]
+              }
+            }
+          }
+        } else {
+          if (ctx?.paging?.startIndex > 1 && !this._serviceClient[baseEntity]?.nextLink[url]) { // no previous paging and invalid startIndex
+            ctx.paging.totalResults = ctx.paging.startIndex - 1
+            return { body: { value: [] } }
+          }
+        }
 
         // execute request
         const f = await fetch(url, options)
@@ -708,30 +733,50 @@ export class HelperRest {
           throw new Error(JSON.stringify(result))
         }
         this.scimgateway.logDebug(baseEntity, `doRequest ${method} ${options.protocol}//${options.host}${(options.port ? `:${options.port}` : '')}${options.path} Body = ${JSON.stringify(body)} Response = ${JSON.stringify(result)}`)
-        if (result.body && typeof result.body === 'object' && result.body['@odata.nextLink']) { // {"@odata.nextLink": "https://graph.microsoft.com/beta/users?$top=100&$skiptoken=xxx"}
-          // OData paging
-          const nextUrl = result.body['@odata.nextLink'].split('?')[1] // keep search query
-          const arr = result['@odata.nextLink'].split('?')[0].split('/')
-          const objType = (arr[arr.length - 1]) // users
-          let startIndexNext = ''
-          if (this._serviceClient[baseEntity].nextLink[objType]) {
-            for (const k in this._serviceClient[baseEntity].nextLink[objType]) {
-              if (this._serviceClient[baseEntity].nextLink[objType][k] === nextUrl) return result // repetive startIndex=1
-              startIndexNext = k
-              break
+
+        // OData paging logic
+        // client prerequisite for enabling doRequest() OData paging support (see plugin-entra-id):
+        //   let paging = { startIndex: getObj.startIndex }
+        //   if (!ctx) ctx = { paging }
+        //   else ctx.paging = paging
+        if (result.body && typeof result.body === 'object') {
+          if (result.body['@odata.nextLink']) { // {"@odata.nextLink": "https://graph.microsoft.com/beta/users?$top=100&$skiptoken=xxx"}
+            if (!ctx) ctx = {}
+            if (!ctx.paging) ctx.paging = {}
+            const nextLinkBase = decodeURIComponent(result.body['@odata.nextLink'].substring(0, result.body['@odata.nextLink'].indexOf('$skiptoken') - 1))
+            const count = result.body['@odata.count']
+            if (count !== undefined) {
+              ctx.paging.totalResults = count
+            }
+            let totalResults = ctx.paging.totalResults
+            if (!totalResults) totalResults = (this._serviceClient[baseEntity].nextLink[nextLinkBase]?.totalResults)
+            let isCount = this._serviceClient[baseEntity].nextLink[nextLinkBase]?.isCount || count !== undefined
+            const itemsPerPage = result.body.value.length
+            this._serviceClient[baseEntity].nextLink[nextLinkBase] = {}
+            this._serviceClient[baseEntity].nextLink[nextLinkBase]['startIndex'] = ctx.paging.startIndex ? ctx.paging.startIndex + itemsPerPage : itemsPerPage + 1
+            this._serviceClient[baseEntity].nextLink[nextLinkBase]['@odata.nextLink'] = result.body['@odata.nextLink']
+            this._serviceClient[baseEntity].nextLink[nextLinkBase]['isCount'] = isCount
+            if (isCount) {
+              this._serviceClient[baseEntity].nextLink[nextLinkBase]['totalResults'] = totalResults // count=true ignored when using nextLink
+              ctx.paging.totalResults = totalResults
+            } else {
+              const totalResults = ctx.paging.startIndex - 1 + (itemsPerPage * 2) // ensure new client paging
+              this._serviceClient[baseEntity].nextLink[nextLinkBase]['totalResults'] = totalResults
+              ctx.paging.totalResults = totalResults
+            }
+          } else { // no more paging
+            const linkBase = decodeURIComponent(url.substring(0, url.indexOf('$skiptoken') - 1))
+            if (ctx?.paging?.startIndex && ctx.paging.startIndex > 1 && this._serviceClient[baseEntity]?.nextLink[linkBase]) {
+              if (!this._serviceClient[baseEntity]?.nextLink[linkBase].isCount) { // final no count page
+                const itemsPerPage = result.body.value.length
+                const totalResults = ctx.paging.startIndex - 1 + itemsPerPage
+                this._serviceClient[baseEntity].nextLink[linkBase]['totalResults'] = totalResults
+                ctx.paging.totalResults = totalResults
+              }
             }
           }
-          const a = result.body['@odata.nextLink'].split('top=')
-          let top = '0'
-          if (a.length > 1) {
-            top = a[1].split('&')[0]
-          }
-          if (!startIndexNext) startIndexNext = (Number(top) + 1).toString()
-          else startIndexNext = (Number(startIndexNext) + Number(top) + 1).toString()
-          // reset and set new nextLink
-          this._serviceClient[baseEntity].nextLink[objType] = {}
-          this._serviceClient[baseEntity].nextLink[objType][startIndexNext] = nextUrl
         }
+
         return result
       } finally {
         clearTimeout(timeout)
@@ -976,36 +1021,6 @@ export class HelperRest {
     return await this.doRequestHandler(baseEntity, method, path, body, ctx, opt)
   }
 
-  /**
-  * nextLinkPaging returns paging url when using OData e.g., Entra ID 
-  * @param baseEntity baseEntity
-  * @param objType e.g., 'users' or 'groups', a type that corresponds with what's being used by endpoint url request
-  * @param startIndex SCIM startIndex paramenter
-  * @returns paging url to be used
-  **/
-  public nextLinkPaging(baseEntity: string, objType: string, startIndex: number) {
-    objType = objType.toLowerCase() // users or groups
-    let nextPath = ''
-    if (!startIndex || !this._serviceClient[baseEntity]) return ''
-    if (startIndex < 2) {
-      if (this._serviceClient[baseEntity].nextLink[objType]) {
-        this._serviceClient[baseEntity].nextLink[objType] = null
-      }
-      return ''
-    }
-    if (this._serviceClient[baseEntity].nextLink[objType]) {
-      if (this._serviceClient[baseEntity].nextLink[objType][startIndex]) {
-        nextPath = `/users?${this._serviceClient[baseEntity].nextLink[objType][startIndex]}`
-      } else {
-        this._serviceClient[baseEntity].nextLink[objType] = null
-        return ''
-      }
-    } else {
-      return ''
-    }
-    return nextPath
-  }
-
   /**
   * getGraphUrl returns Microsoft Graph API url used for Entra ID 
   * @returns Microsoft Graph API url

package/lib/plugin-entra-id.ts

@@ -101,9 +101,17 @@ if (config.map) { // having licensDetails map here instead of config file
 }
 
 const userAttributes: string[] = []
-for (const key in config.map.user) { // userAttributes = ['country', 'preferredLanguage', 'mail', 'city', 'displayName', 'postalCode', 'jobTitle', 'businessPhones', 'onPremisesSyncEnabled', 'officeLocation', 'name.givenName', 'passwordPolicies', 'id', 'state', 'department', 'mailNickname', 'manager.managerId', 'active', 'userName', 'name.familyName', 'proxyAddresses.value', 'servicePlan.value', 'mobilePhone', 'streetAddress', 'onPremisesImmutableId', 'userType', 'usageLocation']
+for (const key in config.map.user) { // userAttributes = ['id', 'country', 'preferredLanguage', 'mail', 'city', 'displayName', 'postalCode', 'jobTitle', 'businessPhones', 'onPremisesSyncEnabled', 'officeLocation', 'name.givenName', 'passwordPolicies', 'id', 'state', 'department', 'mailNickname', 'manager.managerId', 'active', 'userName', 'name.familyName', 'proxyAddresses.value', 'servicePlan.value', 'mobilePhone', 'streetAddress', 'onPremisesImmutableId', 'userType', 'usageLocation']
   if (config.map.user[key].mapTo) userAttributes.push(config.map.user[key].mapTo)
 }
+if (!userAttributes.includes('id')) userAttributes.push('id')
+
+const groupAttributes: string[] = []
+for (const key in config.map.group) { // groupAttributes = ['id', 'displayName', 'securityEnabled', 'mailEnabled']
+  if (config.map.group[key].mapTo) groupAttributes.push(config.map.group[key].mapTo)
+}
+if (!groupAttributes.includes('id')) groupAttributes.push('id')
+if (!groupAttributes.includes('members.value')) groupAttributes.push('members.value')
 
 // =================================================
 // getUsers
@@ -150,18 +158,17 @@ scimgateway.getUsers = async (baseEntity, getObj, attributes, ctx) => {
     throw new Error(`${action} error: not supporting advanced filtering: ${getObj.rawFilter}`)
   } else {
     // mandatory - no filtering (!getObj.operator && !getObj.rawFilter) - all users to be returned - correspond to exploreUsers() in versions < 4.x.x
-    if (getObj.startIndex && getObj.startIndex > 1) { // paging
-      path = helper.nextLinkPaging(baseEntity, 'users', getObj.startIndex)
-      if (!path) return ret
-    } else {
-      getObj.count = (!getObj.count || getObj.count > 999) ? 999 : getObj.count
-      path = `/users?$top=${getObj.count}` // paging not supported using filter (Entra ID default page=100, max=999)
-    }
+    path = `/users?$top=${getObj.count}&$count=true`
   }
   // mandatory if-else logic - end
 
   if (!path) throw new Error(`${action} error: mandatory if-else logic not fully implemented`)
 
+  // enable doRequest() OData paging support 
+  let paging = { startIndex: getObj.startIndex }
+  if (!ctx) ctx = { paging }
+  else ctx.paging = paging
+
   try {
     let response: any
     if (path === 'getUser') { // special
@@ -176,10 +183,16 @@ scimgateway.getUsers = async (baseEntity, getObj, attributes, ctx) => {
       const [scimObj] = scimgateway.endpointMapper('inbound', response.body.value[i], config.map.user) // endpoint => SCIM/CustomSCIM attribute standard
       if (scimObj && typeof scimObj === 'object' && Object.keys(scimObj).length > 0) ret.Resources.push(scimObj)
     }
-    if (getObj.count === response.body.value.length) ret.totalResults = 99999999 // to ensure we get a new paging request - don't know the total numbers of users - metadata directoryObject collections are not countable
+
+    if (getObj.startIndex !== ctx.paging.startIndex) { // changed by doRequest()
+      ret.startIndex = ctx.paging.startIndex
+    }
+    if (ctx.paging.totalResults) ret.totalResults = ctx.paging.totalResults // set by doRequest()
     else ret.totalResults = getObj.startIndex ? getObj.startIndex - 1 + response.body.value.length : response.body.value.length
+
     return (ret)
   } catch (err: any) {
+    if (err.message.includes('Request_ResourceNotFound')) return { Resources: [] }
     throw new Error(`${action} error: ${err.message}`)
   }
 }
@@ -329,9 +342,7 @@ scimgateway.getGroups = async (baseEntity, getObj, attributes, ctx) => {
     totalResults: null,
   }
 
-  if (attributes.length < 1) attributes = ['id', 'displayName', 'members.value']
-  if (!attributes.includes('id')) attributes.push('id')
-
+  if (attributes.length === 0) attributes = groupAttributes
   let includeMembers = false
   if (attributes.includes('members.value') || attributes.includes('members')) {
     includeMembers = true
@@ -356,9 +367,7 @@ scimgateway.getGroups = async (baseEntity, getObj, attributes, ctx) => {
     } else if (getObj.operator === 'eq' && getObj.attribute === 'members.value') {
       // mandatory - return all groups the user 'id' (getObj.value) is member of - correspond to getGroupMembers() in versions < 4.x.x
       // Resources = [{ id: <id-group>> , displayName: <displayName-group>, members [{value: <id-user>}] }]
-      // not using below expand because Entra ID returns only a maximum of 20 items for the expanded relationship
-      // path = `/users/${getObj.value}/memberOf/microsoft.graph.group?$select=id,displayName&$expand=members($select=id,displayName)`
-      path = `/users/${getObj.value}/memberOf/microsoft.graph.group?$select=id,displayName`
+      path = `/users/${getObj.value}/memberOf/microsoft.graph.group?$top=${getObj.count}&$count=true&select=id,displayName`
     } else {
       // optional - simpel filtering
       throw new Error(`${action} error: not supporting simpel filtering: ${getObj.rawFilter}`)
@@ -368,16 +377,18 @@ scimgateway.getGroups = async (baseEntity, getObj, attributes, ctx) => {
     throw new Error(`${action} error: not supporting advanced filtering: ${getObj.rawFilter}`)
   } else {
     // mandatory - no filtering (!getObj.operator && !getObj.rawFilter) - all groups to be returned - correspond to exploreGroups() in versions < 4.x.x
-    // Entra paging not supported because of select filter (Entra default page=100, max=999)
-    // TODO: use a query that supports paging to fix current 999 limit of groups
-    getObj.count = 999
-    if (includeMembers) path = `/groups?$top=${getObj.count}&$select=${attrs.join()}&$expand=members($select=id,displayName)`
-    else path = `/groups?$top=${getObj.count}&$select=${attrs.join()}`
+    if (includeMembers) path = `/groups?$top=${getObj.count}&$count=true&$select=${attrs.join()}&$expand=members($select=id,displayName)`
+    else path = `/groups?$top=${getObj.count}&$count=true&$select=${attrs.join()}`
   }
   // mandatory if-else logic - end
 
   if (!path) throw new Error(`${action} error: mandatory if-else logic not fully implemented`)
 
+  // enable doRequest() OData paging support 
+  let paging = { startIndex: getObj.startIndex }
+  if (!ctx) ctx = { paging }
+  else ctx.paging = paging
+
   try {
     let response = await helper.doRequest(baseEntity, method, path, body, ctx)
     if (!response.body) {
@@ -411,12 +422,15 @@ scimgateway.getGroups = async (baseEntity, getObj, attributes, ctx) => {
       }
     }
 
-    // Entra paging not supported because of select filter
-    getObj.startIndex = 1
-    ret.totalResults = getObj.startIndex ? getObj.startIndex - 1 + response.body.value.length : response.body.value.length
+    if (getObj.startIndex !== ctx.paging.startIndex) { // changed by doRequest()
+      ret.startIndex = ctx.paging.startIndex
+    }
+    if (ctx.paging.totalResults) ret.totalResults = ctx.paging.totalResults // set by doRequest()
+    else ret.totalResults = getObj.startIndex ? getObj.startIndex - 1 + response.body.value.length : response.body.value.length
 
     return (ret)
   } catch (err: any) {
+    if (err.message.includes('Request_ResourceNotFound')) return { Resources: [] }
     throw new Error(`${action} error: ${err.message}`)
   }
 }
@@ -427,6 +441,7 @@ scimgateway.getGroups = async (baseEntity, getObj, attributes, ctx) => {
 scimgateway.createGroup = async (baseEntity, groupObj, ctx) => {
   const action = 'createGroup'
   scimgateway.logDebug(baseEntity, `handling ${action} groupObj=${JSON.stringify(groupObj)} passThrough=${ctx ? 'true' : 'false'}`)
+
   const body: any = { displayName: groupObj.displayName }
   body.mailNickName = groupObj.displayName
   body.mailEnabled = false
@@ -454,7 +469,12 @@ scimgateway.createGroup = async (baseEntity, groupObj, ctx) => {
 scimgateway.deleteGroup = async (baseEntity, id, ctx) => {
   const action = 'deleteGroup'
   scimgateway.logDebug(baseEntity, `handling ${action} id=${id} passThrough=${ctx ? 'true' : 'false'}`)
-  throw new Error(`${action} error: ${action} is not supported`)
+
+  const method = 'DELETE'
+  const path = `/groups/${id}`
+  const body = null
+
+  await helper.doRequest(baseEntity, method, path, body, ctx)
 }
 
 // =================================================

package/lib/plugin-scim.ts

@@ -87,10 +87,10 @@ scimgateway.getUsers = async (baseEntity, getObj, attributes, ctx) => {
       else if (typeof (response.body) === 'object' && Object.keys(response.body).length > 0) responseArr = [response.body]
     }
 
-    if (!getObj.startIndex && !getObj.count) { // client request without paging
-      getObj.startIndex = 1
-      getObj.count = responseArr.length
-    }
+    // no paging support
+    if (!getObj.startIndex) getObj.startIndex = 1
+    if (!getObj.count) getObj.count = responseArr.length
+    if (getObj.count > responseArr.length) getObj.count = responseArr.length
 
     for (let i = 0; i < responseArr.length && (i + 1 - getObj.startIndex) < getObj.count; ++i) {
       const userObj: any = responseArr[i]
@@ -340,10 +340,10 @@ scimgateway.getGroups = async (baseEntity, getObj, attributes, ctx) => {
       else if (typeof (response.body) === 'object' && Object.keys(response.body).length > 0) responseArr = [response.body]
     }
 
-    if (!getObj.startIndex && !getObj.count) { // client request without paging
-      getObj.startIndex = 1
-      getObj.count = responseArr.length
-    }
+    // no paging support
+    if (!getObj.startIndex) getObj.startIndex = 1
+    if (!getObj.count) getObj.count = responseArr.length
+    if (getObj.count > responseArr.length) getObj.count = responseArr.length
 
     for (let i = 0; i < responseArr.length && (i + 1 - getObj.startIndex) < getObj.count; ++i) {
       const groupObj = responseArr[i]

package/lib/scimgateway.ts

@@ -854,7 +854,7 @@ export class ScimGateway {
     const getHandlerSchemas = async (ctx: Context) => {
       let tx = this.scimDef.Schemas
       tx = utilsScim.addResources(tx, undefined, undefined, undefined)
-      tx = utilsScim.addSchemas(tx, isScimv2, undefined, undefined)
+      tx = utilsScim.addSchemasStripAttr(tx, isScimv2)
       ctx.response.body = JSON.stringify(tx)
     }
     funcHandler.getHandlerSchemas = getHandlerSchemas
@@ -1208,7 +1208,7 @@ export class ScimGateway {
         }
 
         scimdata = utils.stripObj(obj, ctx.query.attributes, ctx.query.excludedAttributes)
-        scimdata = utilsScim.addSchemas(scimdata, isScimv2, handle.description, undefined)
+        scimdata = utilsScim.addSchemasStripAttr(scimdata, isScimv2, handle.description)
 
         if (!this.config.scimgateway.scim.skipMetaLocation) {
           const location = ctx.origin + ctx.path
@@ -1365,10 +1365,8 @@ export class ScimGateway {
       if (getObj.operator === 'eq' && ['id', 'userName', 'externalId', 'displayName', 'members.value'].includes(getObj.attribute)) info = ` ${getObj.attribute}=${getObj.value}`
       logger.debug(`${gwName} [Get ${handle.description}s]${info}`, { baseEntity: ctx?.routeObj?.baseEntity })
       try {
-        getObj.startIndex = ctx.query.startIndex ? parseInt(ctx.query.startIndex) : undefined
-        getObj.count = ctx.query.count ? parseInt(ctx.query.count) : undefined
-        if (getObj.startIndex && !getObj.count) getObj.count = 200 // defaults to 200 (plugin may override)
-        if (getObj.count && !getObj.startIndex) getObj.startIndex = 1
+        getObj.startIndex = ctx.query.startIndex ? parseInt(ctx.query.startIndex, 10) : 1
+        getObj.count = ctx.query.count ? parseInt(ctx.query.count, 10) : 200 // defaults to 200 (plugin may override)
 
         let res: any
         const obj: any = utils.copyObj(getObj)
@@ -1435,16 +1433,17 @@ export class ScimGateway {
             }
           }
         }
-        let scimdata: { [key: string]: any } = {
-          Resources: [],
-          totalResults: null,
-        }
-        if (res) {
-          if (res.Resources && Array.isArray(res.Resources)) {
-            scimdata.Resources = res.Resources
-            scimdata.totalResults = res.totalResults
-          } else if (Array.isArray(res)) scimdata.Resources = res
-          else if (typeof (res) === 'object' && Object.keys(res).length > 0) scimdata.Resources[0] = res
+
+        let location: string | undefined = ctx.origin + ctx.path
+        if (this.config.scimgateway.scim.skipMetaLocation) location = undefined
+        else if (ctx.query.excludedAttributes && ctx.query.excludedAttributes.includes('meta')) location = undefined
+
+        let scimdata = utilsScim.addResources(res, ctx.query.startIndex, ctx.query.sortBy, ctx.query.sortOrder)
+        scimdata = utilsScim.addSchemasStripAttr(scimdata, isScimv2, handle.description, ctx.query.attributes, ctx.query.excludedAttributes, location)
+        if (getObj.count === 0) {
+          scimdata.Resources = []
+          scimdata.itemsPerPage = 0
+          // keep totalResults
         }
 
         if (scimdata.Resources.length === 1) {
@@ -1454,16 +1453,6 @@ export class ScimGateway {
           else if (obj.displayName) ctx.target = obj.displayName
         }
 
-        let location: string | undefined = ctx.origin + ctx.path
-        if (this.config.scimgateway.scim.skipMetaLocation) location = undefined
-        else if (ctx.query.excludedAttributes && ctx.query.excludedAttributes.includes('meta')) location = undefined
-        for (let i = 0; i < scimdata.Resources.length; i++) {
-          utils.getEtag(scimdata.Resources[i])
-          scimdata.Resources[i] = utils.stripObj(scimdata.Resources[i], ctx.query.attributes, ctx.query.excludedAttributes)
-        }
-        scimdata = utilsScim.addResources(scimdata, ctx.query.startIndex, ctx.query.sortBy, ctx.query.sortOrder)
-        scimdata = utilsScim.addSchemas(scimdata, isScimv2, handle.description, location)
-
         ctx.response.body = JSON.stringify(scimdata)
       } catch (err: any) {
         if (isScimv2) ctx.response.status = 400
@@ -1593,7 +1582,7 @@ export class ScimGateway {
           if (!jsonBody.meta) jsonBody.meta = {}
           jsonBody.meta.location = location
         }
-        jsonBody = utilsScim.addSchemas(jsonBody, isScimv2, handle.description, undefined)
+        jsonBody = utilsScim.addSchemasStripAttr(jsonBody, isScimv2, handle.description)
         if (eTag) ctx.response.headers.set('ETag', eTag)
         if (jsonBody?.meta?.location) ctx.response.headers.set('Location', jsonBody.meta.location)
         ctx.response.status = 201
@@ -1731,7 +1720,7 @@ export class ScimGateway {
         }
 
         scimres = utils.stripObj(userObj, ctx.query.attributes, ctx.query.excludedAttributes)
-        scimres = utilsScim.addSchemas(scimres, isScimv2, handle.description, undefined)
+        scimres = utilsScim.addSchemasStripAttr(scimres, isScimv2, handle.description)
         if (eTag) ctx.response.headers.set('ETag', eTag)
         if (scimres?.meta?.location) ctx.response.headers.set('Location', scimres.meta.location)
         ctx.response.status = 200
@@ -2560,23 +2549,34 @@ export class ScimGateway {
       if (typeof (this as any)[handler.groups.getMethod] !== 'function') return groups // method not implemented
       if (this.config.scimgateway.scim.groupMemberOfUser) return groups // only support user member of group
       let res: any
-      try {
-        const ob = { attribute: 'members.value', operator: 'eq', value: decodeURIComponent(id) }
-        const attributes = ['id', 'displayName']
-        logger.debug(`${gwName} calling ${handler.groups.getMethod} - groups to be included`, { baseEntity })
-        res = await (this as any)[handler.groups.getMethod](baseEntity, ob, attributes, ctxPassThrough)
-      } catch (err) { void 0 }
-      if (res && res.Resources && Array.isArray(res.Resources) && res.Resources.length > 0) {
-        for (let i = 0; i < res.Resources.length; i++) {
-          if (!res.Resources[i].id) continue
-          const el: any = {}
-          el.value = res.Resources[i].id
-          if (res.Resources[i].displayName) el.display = res.Resources[i].displayName
-          if (isScimv2) el.type = 'direct'
-          else el.type = { value: 'direct' }
-          groups.push(el) // { "value": "Admins", "display": "Admins", "type": "direct"}
+      const ob: Record<string, any> = { attribute: 'members.value', operator: 'eq', value: decodeURIComponent(id) }
+      const attributes = ['id', 'displayName']
+      const count = 200
+      let startIndex = 1
+      let nextStartIndex = 1
+      do {
+        try {
+          logger.debug(`${gwName} calling ${handler.groups.getMethod} - groups to be included`, { baseEntity })
+          startIndex = nextStartIndex
+          ob.startIndex = startIndex
+          ob.count = count
+          res = await (this as any)[handler.groups.getMethod](baseEntity, ob, attributes, ctxPassThrough)
+        } catch (err) { void 0 }
+        if (res && res.Resources) {
+          if (Array.isArray(res.Resources) && res.Resources.length > 0) {
+            for (let i = 0; i < res.Resources.length; i++) {
+              if (!res.Resources[i].id) continue
+              const el: any = {}
+              el.value = res.Resources[i].id
+              if (res.Resources[i].displayName) el.display = res.Resources[i].displayName
+              if (isScimv2) el.type = 'direct'
+              else el.type = { value: 'direct' }
+              groups.push(el) // { "value": "Admins", "display": "Admins", "type": "direct"}
+            }
+            nextStartIndex = utilsScim.getNextStartIndex(res.totalResults, startIndex, res.Resources.length)
+          }
         }
-      }
+      } while (nextStartIndex > startIndex)
       return groups
     }
     this.getMemberOf = getMemberOf

package/lib/utils-scim.ts

@@ -515,8 +515,7 @@ export function endpointMapper(direction: string, parseObj: any, mapObj: any) {
               const keyRoot = key2.split('.').slice(0, -1).join('.') // xx.yy.mapTo => xx.yy
               if (dotMap[`${keyRoot}.type`] === 'array' && arrIndex >= 0) {
                 dotNewObj[`${keyRoot}.${arrIndex}`] = dotParse[keyOrg] // servicePlan.0.value => servicePlan.0 and groups[0].value => memberOf.0
-              }
-              dotNewObj[keyRoot] = dotParse[key] // {"accountEnabled": {"mapTo": "active"} => str.replace("accountEnabled", "active")
+              } else dotNewObj[keyRoot] = dotParse[key] // {"accountEnabled": {"mapTo": "active"} => str.replace("accountEnabled", "active")
               break
             }
           }
@@ -768,14 +767,25 @@ export function addResources(data: any, startIndex?: string, sortBy?: string, so
   if (Array.isArray(data)) res.Resources = data
   else if (data.Resources) {
     res.Resources = data.Resources
-    res.totalResults = data.totalResults
   } else res.Resources.push(data)
 
+  if (Object.hasOwn(data, 'totalResults')) res.totalResults = data.totalResults
+  if (data.startIndex) {
+    res.startIndex = data.startIndex
+  } else if (startIndex) {
+    res.startIndex = parseInt(startIndex, 10)
+  } else {
+    res.startIndex = 1
+  }
+
   // pagination
   if (!res.totalResults) res.totalResults = res.Resources.length // Specifies the total number of results matching the Consumer query
   res.itemsPerPage = res.Resources.length // Specifies the number of search results returned in a query response page
-  if (startIndex) res.startIndex = parseInt(startIndex) // The 1-based index of the first result in the current set of search results
-  else res.startIndex = 1
+
+  if (!res.startIndex || isNaN(res.startIndex)) {
+    if (startIndex) res.startIndex = parseInt(startIndex) // The 1-based index of the first result in the current set of search results
+    else res.startIndex = 1
+  }
   if (res.startIndex > res.totalResults) { // invalid paging request
     res.Resources = []
     res.itemsPerPage = 0
@@ -785,7 +795,7 @@ export function addResources(data: any, startIndex?: string, sortBy?: string, so
   return res
 }
 
-export function addSchemas(data: Record<string, any>, isScimv2: boolean, type?: string, location?: string) {
+export function addSchemasStripAttr(data: Record<string, any>, isScimv2: boolean, type?: string, attributes?: string, excludedAttributes?: string, location?: string) {
   if (!type) {
     if (isScimv2) data.schemas = ['urn:ietf:params:scim:api:messages:2.0:ListResponse']
     else data.schemas = ['urn:scim:schemas:core:1.0']
@@ -796,6 +806,9 @@ export function addSchemas(data: Record<string, any>, isScimv2: boolean, type?:
     if (isScimv2) data.schemas = ['urn:ietf:params:scim:api:messages:2.0:ListResponse']
     else data.schemas = ['urn:scim:schemas:core:1.0']
     for (let i = 0; i < data.Resources.length; i++) {
+      utils.getEtag(data.Resources[i])
+      data.Resources[i] = utils.stripObj(data.Resources[i], attributes, excludedAttributes)
+
       if (isScimv2) { // scim v2 add schemas/resourceType on each element
         if (type === 'User') {
           const val = 'urn:ietf:params:scim:schemas:core:2.0:User'
@@ -1062,3 +1075,14 @@ export function loadScimDef(version: ScimVersion, customPath?: string): any {
   }
   return v === '1.1' ? scimdefV1Default : scimdefV2Default
 }
+
+/**
+* returns next pagination startIndex
+* pagination should stop when nextStartIndex <= startIndex
+*/
+export function getNextStartIndex(totalResults: number, startIndex: number, itemsPerPage: number): number {
+  if (totalResults === undefined || startIndex == undefined || itemsPerPage === undefined) return startIndex
+  const nextStartIndex = startIndex + itemsPerPage
+  if (nextStartIndex >= totalResults) return startIndex
+  return nextStartIndex
+}

package/lib/utils.ts

@@ -744,7 +744,6 @@ export class TimerMapCache {
   createItem(item: string): string {
     if (!item) return ''
     this.itemCache.set(item, setTimeout(() => {
-      console.log('deleting item')
       this.itemCache.delete(item)
     },
     this.itemTimeout))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "scimgateway",
-  "version": "6.1.3",
+  "version": "6.1.4",
   "type": "module",
   "description": "Using SCIM protocol as a gateway for user provisioning to other endpoints",
   "author": "Jarle Elshaug <jarle.elshaug@gmail.com> (https://elshaug.xyz)",