renovate 44.45.1 → 44.45.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -10,6 +10,21 @@ import { GithubGraphqlPackageCacheStrategy } from "./cache-strategies/package-ca
|
|
|
10
10
|
import is from "@sindresorhus/is";
|
|
11
11
|
//#region lib/util/github/graphql/datasource-fetcher.ts
|
|
12
12
|
/**
|
|
13
|
+
* Page sizes to try, from the largest to the smallest.
|
|
14
|
+
*
|
|
15
|
+
* Repositories with many tags can exceed the GitHub GraphQL execution
|
|
16
|
+
* timeout even for small page sizes, hence the low floor.
|
|
17
|
+
*
|
|
18
|
+
* @see https://github.com/renovatebot/renovate/discussions/45248
|
|
19
|
+
*/
|
|
20
|
+
const pageSizes = [
|
|
21
|
+
100,
|
|
22
|
+
50,
|
|
23
|
+
25,
|
|
24
|
+
10,
|
|
25
|
+
5
|
|
26
|
+
];
|
|
27
|
+
/**
|
|
13
28
|
* We know empirically that certain type of GraphQL errors
|
|
14
29
|
* can be fixed by shrinking page size.
|
|
15
30
|
*
|
|
@@ -29,7 +44,7 @@ var GithubGraphqlDatasourceFetcher = class GithubGraphqlDatasourceFetcher {
|
|
|
29
44
|
baseUrl;
|
|
30
45
|
repoOwner;
|
|
31
46
|
repoName;
|
|
32
|
-
itemsPerQuery =
|
|
47
|
+
itemsPerQuery = pageSizes[0];
|
|
33
48
|
queryCount = 0;
|
|
34
49
|
cursor = null;
|
|
35
50
|
isPersistent;
|
|
@@ -94,15 +109,10 @@ var GithubGraphqlDatasourceFetcher = class GithubGraphqlDatasourceFetcher {
|
|
|
94
109
|
return [data.repository.payload, null];
|
|
95
110
|
}
|
|
96
111
|
shrinkPageSize() {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if (this.itemsPerQuery === 50) {
|
|
102
|
-
this.itemsPerQuery = 25;
|
|
103
|
-
return true;
|
|
104
|
-
}
|
|
105
|
-
return false;
|
|
112
|
+
const nextIdx = pageSizes.indexOf(this.itemsPerQuery) + 1;
|
|
113
|
+
if (nextIdx >= pageSizes.length) return false;
|
|
114
|
+
this.itemsPerQuery = pageSizes[nextIdx];
|
|
115
|
+
return true;
|
|
106
116
|
}
|
|
107
117
|
hasReachedQueryLimit() {
|
|
108
118
|
return this.queryCount >= 100;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"datasource-fetcher.js","names":["packageCache.get","packageCache.set","memCache.get"],"sources":["../../../../lib/util/github/graphql/datasource-fetcher.ts"],"sourcesContent":["import is from '@sindresorhus/is';\nimport { GlobalConfig } from '../../../config/global.ts';\nimport { logger } from '../../../logger/index.ts';\nimport { ExternalHostError } from '../../../types/errors/external-host-error.ts';\nimport * as memCache from '../../cache/memory/index.ts';\nimport * as packageCache from '../../cache/package/index.ts';\nimport type { PackageCacheNamespace } from '../../cache/package/types.ts';\nimport type {\n GithubGraphqlResponse,\n GithubHttp,\n GithubHttpOptions,\n} from '../../http/github.ts';\nimport type { HttpResponse } from '../../http/types.ts';\nimport { regEx } from '../../regex.ts';\nimport { getApiBaseUrl } from '../url.ts';\nimport { GithubGraphqlMemoryCacheStrategy } from './cache-strategies/memory-cache-strategy.ts';\nimport { GithubGraphqlPackageCacheStrategy } from './cache-strategies/package-cache-strategy.ts';\nimport type {\n GithubDatasourceItem,\n GithubGraphqlCacheStrategy,\n GithubGraphqlDatasourceAdapter,\n GithubGraphqlPayload,\n GithubGraphqlRepoParams,\n GithubGraphqlRepoResponse,\n GithubPackageConfig,\n RawQueryResponse,\n} from './types.ts';\n\n/**\n * We know empirically that certain type of GraphQL errors\n * can be fixed by shrinking page size.\n *\n * @see https://github.com/renovatebot/renovate/issues/16343\n */\nfunction isUnknownGraphqlError(err: Error): boolean {\n const { message } = err;\n return message.startsWith('Something went wrong while executing your query.');\n}\n\nfunction canBeSolvedByShrinking(err: Error): boolean {\n const errors: Error[] = err instanceof AggregateError ? err.errors : [err];\n return errors.some(\n (e) => err instanceof ExternalHostError || isUnknownGraphqlError(e),\n );\n}\n\nexport class GithubGraphqlDatasourceFetcher<\n GraphqlItem,\n ResultItem extends GithubDatasourceItem,\n> {\n static async query<T, U extends GithubDatasourceItem>(\n config: GithubPackageConfig,\n http: GithubHttp,\n adapter: GithubGraphqlDatasourceAdapter<T, U>,\n ): Promise<U[]> {\n const instance = new GithubGraphqlDatasourceFetcher<T, U>(\n config,\n http,\n adapter,\n );\n const items = await instance.getItems();\n return items;\n }\n\n private readonly baseUrl: string;\n private readonly repoOwner: string;\n private readonly repoName: string;\n\n private itemsPerQuery: 100 | 50 | 25 = 100;\n\n private queryCount = 0;\n\n private cursor: string | null = null;\n\n private isPersistent: boolean | undefined;\n private http: GithubHttp;\n private datasourceAdapter: GithubGraphqlDatasourceAdapter<\n GraphqlItem,\n ResultItem\n >;\n\n constructor(\n packageConfig: GithubPackageConfig,\n http: GithubHttp,\n datasourceAdapter: GithubGraphqlDatasourceAdapter<GraphqlItem, ResultItem>,\n ) {\n this.http = http;\n this.datasourceAdapter = datasourceAdapter;\n const { packageName, registryUrl } = packageConfig;\n [this.repoOwner, this.repoName] = packageName.split('/');\n this.baseUrl = getApiBaseUrl(registryUrl).replace(regEx(/\\/v3\\/$/), '/'); // Replace for GHE\n }\n\n private getCacheNs(): PackageCacheNamespace {\n return this.datasourceAdapter.key;\n }\n\n private getCacheKey(): string {\n return [this.baseUrl, this.repoOwner, this.repoName].join(':');\n }\n\n private getRawQueryOptions(): GithubHttpOptions {\n const baseUrl = this.baseUrl;\n const repository = `${this.repoOwner}/${this.repoName}`;\n const query = this.datasourceAdapter.query;\n const variables: GithubGraphqlRepoParams = {\n owner: this.repoOwner,\n name: this.repoName,\n count: this.itemsPerQuery,\n cursor: this.cursor,\n };\n\n return {\n baseUrl,\n repository,\n readOnly: true,\n body: { query, variables },\n };\n }\n\n private async doRawQuery(): Promise<\n RawQueryResponse<GithubGraphqlPayload<GraphqlItem>>\n > {\n const requestOptions = this.getRawQueryOptions();\n\n type GraphqlData = GithubGraphqlRepoResponse<GraphqlItem>;\n type HttpBody = GithubGraphqlResponse<GraphqlData>;\n let httpRes: HttpResponse<HttpBody>;\n try {\n httpRes = await this.http.postJson<HttpBody>('/graphql', requestOptions);\n } catch (err) {\n return [null, err];\n }\n\n const { body } = httpRes;\n const { data, errors } = body;\n\n if (errors?.length) {\n if (errors.length === 1) {\n const { message } = errors[0];\n const err = new Error(message);\n return [null, err];\n }\n const errorInstances = errors.map(({ message }) => new Error(message));\n const err = new AggregateError(errorInstances);\n return [null, err];\n }\n\n if (!data) {\n const msg = 'GitHub GraphQL datasource: failed to obtain data';\n const err = new Error(msg);\n return [null, err];\n }\n\n if (!data.repository) {\n const msg = 'GitHub GraphQL datasource: failed to obtain repository data';\n const err = new Error(msg);\n return [null, err];\n }\n\n if (!data.repository.payload) {\n const msg =\n 'GitHub GraphQL datasource: failed to obtain repository payload data';\n const err = new Error(msg);\n return [null, err];\n }\n\n this.queryCount += 1;\n\n // For values other than explicit `false`,\n // we assume that items can not be cached.\n this.isPersistent ??= data.repository.isRepoPrivate === false;\n\n const res = data.repository.payload;\n return [res, null];\n }\n\n private shrinkPageSize(): boolean {\n if (this.itemsPerQuery === 100) {\n this.itemsPerQuery = 50;\n return true;\n }\n\n if (this.itemsPerQuery === 50) {\n this.itemsPerQuery = 25;\n return true;\n }\n\n return false;\n }\n\n private hasReachedQueryLimit(): boolean {\n return this.queryCount >= 100;\n }\n\n private async doShrinkableQuery(): Promise<\n GithubGraphqlPayload<GraphqlItem>\n > {\n let res: GithubGraphqlPayload<GraphqlItem> | null = null;\n let err: Error | null = null;\n\n while (!res) {\n [res, err] = await this.doRawQuery();\n if (err) {\n if (!canBeSolvedByShrinking(err)) {\n throw err;\n }\n\n const shrinkResult = this.shrinkPageSize();\n if (!shrinkResult) {\n throw err;\n }\n const { body: _, ...options } = this.getRawQueryOptions();\n logger.debug(\n { options, newSize: this.itemsPerQuery },\n 'Shrinking GitHub GraphQL page size after error',\n );\n }\n }\n\n return res;\n }\n\n private _cacheStrategy: GithubGraphqlCacheStrategy<ResultItem> | undefined;\n\n private cacheStrategy(): GithubGraphqlCacheStrategy<ResultItem> {\n if (this._cacheStrategy) {\n return this._cacheStrategy;\n }\n const cacheNs = this.getCacheNs();\n const cacheKey = this.getCacheKey();\n const cachePrivatePackages = GlobalConfig.get('cachePrivatePackages');\n const skipStabilization = !is.undefined(this.datasourceAdapter.maxItems);\n this._cacheStrategy =\n cachePrivatePackages || this.isPersistent\n ? new GithubGraphqlPackageCacheStrategy<ResultItem>(\n cacheNs,\n cacheKey,\n skipStabilization,\n )\n : new GithubGraphqlMemoryCacheStrategy<ResultItem>(\n cacheNs,\n cacheKey,\n skipStabilization,\n );\n return this._cacheStrategy;\n }\n\n /**\n * This method is responsible for data synchronization.\n * It also detects persistence of the package, based on the first page result.\n */\n private async doPaginatedFetch(): Promise<void> {\n let hasNextPage = true;\n let isPaginationDone = false;\n let nextCursor: string | undefined;\n let itemCount = 0;\n const maxItems = this.datasourceAdapter.maxItems;\n while (hasNextPage && !isPaginationDone && !this.hasReachedQueryLimit()) {\n if (!is.undefined(maxItems) && itemCount >= maxItems) {\n break;\n }\n const queryResult = await this.doShrinkableQuery();\n\n const resultItems: ResultItem[] = [];\n for (const node of queryResult.nodes) {\n const item = this.datasourceAdapter.transform(node);\n if (!item) {\n logger.once.info(\n {\n packageName: `${this.repoOwner}/${this.repoName}`,\n baseUrl: this.baseUrl,\n },\n `GitHub GraphQL datasource: skipping empty item`,\n );\n continue;\n }\n resultItems.push(item);\n }\n itemCount += resultItems.length;\n\n // It's important to call `getCacheStrategy()` after `doShrinkableQuery()`\n // because `doShrinkableQuery()` may change `this.isCacheable`.\n //\n // Otherwise, cache items for public packages will never be persisted\n // in long-term cache.\n isPaginationDone = await this.cacheStrategy().reconcile(resultItems);\n\n hasNextPage = !!queryResult?.pageInfo?.hasNextPage;\n nextCursor = queryResult?.pageInfo?.endCursor;\n if (hasNextPage && nextCursor) {\n this.cursor = nextCursor;\n }\n }\n\n if (this.isPersistent) {\n await this.storePersistenceFlag(30);\n }\n }\n\n private async doCachedQuery(): Promise<ResultItem[]> {\n await this.loadPersistenceFlag();\n if (!this.isPersistent) {\n await this.doPaginatedFetch();\n }\n\n const res = await this.cacheStrategy().finalizeAndReturn();\n if (res.length) {\n return res;\n }\n\n delete this.isPersistent;\n await this.doPaginatedFetch();\n return this.cacheStrategy().finalizeAndReturn();\n }\n\n async loadPersistenceFlag(): Promise<void> {\n const ns = this.getCacheNs();\n const key = `${this.getCacheKey()}:is-persistent`;\n this.isPersistent = await packageCache.get<true>(ns, key);\n }\n\n async storePersistenceFlag(minutes: number): Promise<void> {\n const ns = this.getCacheNs();\n const key = `${this.getCacheKey()}:is-persistent`;\n await packageCache.set(ns, key, true, minutes);\n }\n\n /**\n * This method ensures the only one query is executed\n * to a particular package during single run.\n */\n private doUniqueQuery(): Promise<ResultItem[]> {\n const cacheKey = `github-pending:${this.getCacheNs()}:${this.getCacheKey()}`;\n let resultPromise = memCache.get<Promise<ResultItem[]>>(cacheKey);\n resultPromise ??= this.doCachedQuery();\n memCache.set(cacheKey, resultPromise);\n return resultPromise;\n }\n\n async getItems(): Promise<ResultItem[]> {\n const res = await this.doUniqueQuery();\n return res;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;AAkCA,SAAS,sBAAsB,KAAqB;CAClD,MAAM,EAAE,YAAY;CACpB,OAAO,QAAQ,WAAW,kDAAkD;AAC9E;AAEA,SAAS,uBAAuB,KAAqB;CAEnD,QADwB,eAAe,iBAAiB,IAAI,SAAS,CAAC,GAAG,EAAA,CAC3D,MACX,MAAM,eAAe,qBAAqB,sBAAsB,CAAC,CACpE;AACF;AAEA,IAAa,iCAAb,MAAa,+BAGX;CACA,aAAa,MACX,QACA,MACA,SACc;EAOd,OAAO,MADa,IALC,+BACnB,QACA,MACA,OAEyB,CAAC,CAAC,SAAS;CAExC;CAEA;CACA;CACA;CAEA,gBAAuC;CAEvC,aAAqB;CAErB,SAAgC;CAEhC;CACA;CACA;CAKA,YACE,eACA,MACA,mBACA;EACA,KAAK,OAAO;EACZ,KAAK,oBAAoB;EACzB,MAAM,EAAE,aAAa,gBAAgB;EACrC,CAAC,KAAK,WAAW,KAAK,YAAY,YAAY,MAAM,GAAG;EACvD,KAAK,UAAU,cAAc,WAAW,CAAC,CAAC,QAAQ,MAAM,SAAS,GAAG,GAAG;CACzE;CAEA,aAA4C;EAC1C,OAAO,KAAK,kBAAkB;CAChC;CAEA,cAA8B;EAC5B,OAAO;GAAC,KAAK;GAAS,KAAK;GAAW,KAAK;EAAQ,CAAC,CAAC,KAAK,GAAG;CAC/D;CAEA,qBAAgD;EAW9C,OAAO;GACL,SAXc,KAAK;GAYnB,YAAA,GAXoB,KAAK,UAAU,GAAG,KAAK;GAY3C,UAAU;GACV,MAAM;IAAE,OAZI,KAAK,kBAAkB;IAYpB,WAAA;KAVf,OAAO,KAAK;KACZ,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,QAAQ,KAAK;IAOU;GAAE;EAC3B;CACF;CAEA,MAAc,aAEZ;EACA,MAAM,iBAAiB,KAAK,mBAAmB;EAI/C,IAAI;EACJ,IAAI;GACF,UAAU,MAAM,KAAK,KAAK,SAAmB,YAAY,cAAc;EACzE,SAAS,KAAK;GACZ,OAAO,CAAC,MAAM,GAAG;EACnB;EAEA,MAAM,EAAE,SAAS;EACjB,MAAM,EAAE,MAAM,WAAW;EAEzB,IAAI,QAAQ,QAAQ;GAClB,IAAI,OAAO,WAAW,GAAG;IACvB,MAAM,EAAE,YAAY,OAAO;IAE3B,OAAO,CAAC,MAAM,IADE,MAAM,OACN,CAAC;GACnB;GACA,MAAM,iBAAiB,OAAO,KAAK,EAAE,cAAc,IAAI,MAAM,OAAO,CAAC;GAErE,OAAO,CAAC,MAAM,IADE,eAAe,cACf,CAAC;EACnB;EAEA,IAAI,CAAC,MAGH,OAAO,CAAC,sBAAM,IADE,MAAM,kDACN,CAAC;EAGnB,IAAI,CAAC,KAAK,YAGR,OAAO,CAAC,sBAAM,IADE,MAAM,6DACN,CAAC;EAGnB,IAAI,CAAC,KAAK,WAAW,SAInB,OAAO,CAAC,sBAAM,IADE,MAAM,qEACN,CAAC;EAGnB,KAAK,cAAc;EAInB,KAAK,iBAAiB,KAAK,WAAW,kBAAkB;EAGxD,OAAO,CADK,KAAK,WAAW,SACf,IAAI;CACnB;CAEA,iBAAkC;EAChC,IAAI,KAAK,kBAAkB,KAAK;GAC9B,KAAK,gBAAgB;GACrB,OAAO;EACT;EAEA,IAAI,KAAK,kBAAkB,IAAI;GAC7B,KAAK,gBAAgB;GACrB,OAAO;EACT;EAEA,OAAO;CACT;CAEA,uBAAwC;EACtC,OAAO,KAAK,cAAc;CAC5B;CAEA,MAAc,oBAEZ;EACA,IAAI,MAAgD;EACpD,IAAI,MAAoB;EAExB,OAAO,CAAC,KAAK;GACX,CAAC,KAAK,OAAO,MAAM,KAAK,WAAW;GACnC,IAAI,KAAK;IACP,IAAI,CAAC,uBAAuB,GAAG,GAC7B,MAAM;IAIR,IAAI,CADiB,KAAK,eACV,GACd,MAAM;IAER,MAAM,EAAE,MAAM,GAAG,GAAG,YAAY,KAAK,mBAAmB;IACxD,OAAO,MACL;KAAE;KAAS,SAAS,KAAK;IAAc,GACvC,gDACF;GACF;EACF;EAEA,OAAO;CACT;CAEA;CAEA,gBAAgE;EAC9D,IAAI,KAAK,gBACP,OAAO,KAAK;EAEd,MAAM,UAAU,KAAK,WAAW;EAChC,MAAM,WAAW,KAAK,YAAY;EAClC,MAAM,uBAAuB,aAAa,IAAI,sBAAsB;EACpE,MAAM,oBAAoB,CAAC,GAAG,UAAU,KAAK,kBAAkB,QAAQ;EACvE,KAAK,iBACH,wBAAwB,KAAK,eACzB,IAAI,kCACF,SACA,UACA,iBACF,IACA,IAAI,iCACF,SACA,UACA,iBACF;EACN,OAAO,KAAK;CACd;;;;;CAMA,MAAc,mBAAkC;EAC9C,IAAI,cAAc;EAClB,IAAI,mBAAmB;EACvB,IAAI;EACJ,IAAI,YAAY;EAChB,MAAM,WAAW,KAAK,kBAAkB;EACxC,OAAO,eAAe,CAAC,oBAAoB,CAAC,KAAK,qBAAqB,GAAG;GACvE,IAAI,CAAC,GAAG,UAAU,QAAQ,KAAK,aAAa,UAC1C;GAEF,MAAM,cAAc,MAAM,KAAK,kBAAkB;GAEjD,MAAM,cAA4B,CAAC;GACnC,KAAK,MAAM,QAAQ,YAAY,OAAO;IACpC,MAAM,OAAO,KAAK,kBAAkB,UAAU,IAAI;IAClD,IAAI,CAAC,MAAM;KACT,OAAO,KAAK,KACV;MACE,aAAa,GAAG,KAAK,UAAU,GAAG,KAAK;MACvC,SAAS,KAAK;KAChB,GACA,gDACF;KACA;IACF;IACA,YAAY,KAAK,IAAI;GACvB;GACA,aAAa,YAAY;GAOzB,mBAAmB,MAAM,KAAK,cAAc,CAAC,CAAC,UAAU,WAAW;GAEnE,cAAc,CAAC,CAAC,aAAa,UAAU;GACvC,aAAa,aAAa,UAAU;GACpC,IAAI,eAAe,YACjB,KAAK,SAAS;EAElB;EAEA,IAAI,KAAK,cACP,MAAM,KAAK,qBAAqB,EAAE;CAEtC;CAEA,MAAc,gBAAuC;EACnD,MAAM,KAAK,oBAAoB;EAC/B,IAAI,CAAC,KAAK,cACR,MAAM,KAAK,iBAAiB;EAG9B,MAAM,MAAM,MAAM,KAAK,cAAc,CAAC,CAAC,kBAAkB;EACzD,IAAI,IAAI,QACN,OAAO;EAGT,OAAO,KAAK;EACZ,MAAM,KAAK,iBAAiB;EAC5B,OAAO,KAAK,cAAc,CAAC,CAAC,kBAAkB;CAChD;CAEA,MAAM,sBAAqC;EACzC,MAAM,KAAK,KAAK,WAAW;EAC3B,MAAM,MAAM,GAAG,KAAK,YAAY,EAAE;EAClC,KAAK,eAAe,MAAMA,MAAuB,IAAI,GAAG;CAC1D;CAEA,MAAM,qBAAqB,SAAgC;EACzD,MAAM,KAAK,KAAK,WAAW;EAC3B,MAAM,MAAM,GAAG,KAAK,YAAY,EAAE;EAClC,MAAMC,MAAiB,IAAI,KAAK,MAAM,OAAO;CAC/C;;;;;CAMA,gBAA+C;EAC7C,MAAM,WAAW,kBAAkB,KAAK,WAAW,EAAE,GAAG,KAAK,YAAY;EACzE,IAAI,gBAAgBC,IAAoC,QAAQ;EAChE,kBAAkB,KAAK,cAAc;EACrC,IAAa,UAAU,aAAa;EACpC,OAAO;CACT;CAEA,MAAM,WAAkC;EAEtC,OAAO,MADW,KAAK,cAAc;CAEvC;AACF"}
|
|
1
|
+
{"version":3,"file":"datasource-fetcher.js","names":["packageCache.get","packageCache.set","memCache.get"],"sources":["../../../../lib/util/github/graphql/datasource-fetcher.ts"],"sourcesContent":["import is from '@sindresorhus/is';\nimport { GlobalConfig } from '../../../config/global.ts';\nimport { logger } from '../../../logger/index.ts';\nimport { ExternalHostError } from '../../../types/errors/external-host-error.ts';\nimport * as memCache from '../../cache/memory/index.ts';\nimport * as packageCache from '../../cache/package/index.ts';\nimport type { PackageCacheNamespace } from '../../cache/package/types.ts';\nimport type {\n GithubGraphqlResponse,\n GithubHttp,\n GithubHttpOptions,\n} from '../../http/github.ts';\nimport type { HttpResponse } from '../../http/types.ts';\nimport { regEx } from '../../regex.ts';\nimport { getApiBaseUrl } from '../url.ts';\nimport { GithubGraphqlMemoryCacheStrategy } from './cache-strategies/memory-cache-strategy.ts';\nimport { GithubGraphqlPackageCacheStrategy } from './cache-strategies/package-cache-strategy.ts';\nimport type {\n GithubDatasourceItem,\n GithubGraphqlCacheStrategy,\n GithubGraphqlDatasourceAdapter,\n GithubGraphqlPayload,\n GithubGraphqlRepoParams,\n GithubGraphqlRepoResponse,\n GithubPackageConfig,\n RawQueryResponse,\n} from './types.ts';\n\n/**\n * Page sizes to try, from the largest to the smallest.\n *\n * Repositories with many tags can exceed the GitHub GraphQL execution\n * timeout even for small page sizes, hence the low floor.\n *\n * @see https://github.com/renovatebot/renovate/discussions/45248\n */\nconst pageSizes = [100, 50, 25, 10, 5] as const;\n\ntype PageSize = (typeof pageSizes)[number];\n\n/**\n * We know empirically that certain type of GraphQL errors\n * can be fixed by shrinking page size.\n *\n * @see https://github.com/renovatebot/renovate/issues/16343\n */\nfunction isUnknownGraphqlError(err: Error): boolean {\n const { message } = err;\n return message.startsWith('Something went wrong while executing your query.');\n}\n\nfunction canBeSolvedByShrinking(err: Error): boolean {\n const errors: Error[] = err instanceof AggregateError ? err.errors : [err];\n return errors.some(\n (e) => err instanceof ExternalHostError || isUnknownGraphqlError(e),\n );\n}\n\nexport class GithubGraphqlDatasourceFetcher<\n GraphqlItem,\n ResultItem extends GithubDatasourceItem,\n> {\n static async query<T, U extends GithubDatasourceItem>(\n config: GithubPackageConfig,\n http: GithubHttp,\n adapter: GithubGraphqlDatasourceAdapter<T, U>,\n ): Promise<U[]> {\n const instance = new GithubGraphqlDatasourceFetcher<T, U>(\n config,\n http,\n adapter,\n );\n const items = await instance.getItems();\n return items;\n }\n\n private readonly baseUrl: string;\n private readonly repoOwner: string;\n private readonly repoName: string;\n\n private itemsPerQuery: PageSize = pageSizes[0];\n\n private queryCount = 0;\n\n private cursor: string | null = null;\n\n private isPersistent: boolean | undefined;\n private http: GithubHttp;\n private datasourceAdapter: GithubGraphqlDatasourceAdapter<\n GraphqlItem,\n ResultItem\n >;\n\n constructor(\n packageConfig: GithubPackageConfig,\n http: GithubHttp,\n datasourceAdapter: GithubGraphqlDatasourceAdapter<GraphqlItem, ResultItem>,\n ) {\n this.http = http;\n this.datasourceAdapter = datasourceAdapter;\n const { packageName, registryUrl } = packageConfig;\n [this.repoOwner, this.repoName] = packageName.split('/');\n this.baseUrl = getApiBaseUrl(registryUrl).replace(regEx(/\\/v3\\/$/), '/'); // Replace for GHE\n }\n\n private getCacheNs(): PackageCacheNamespace {\n return this.datasourceAdapter.key;\n }\n\n private getCacheKey(): string {\n return [this.baseUrl, this.repoOwner, this.repoName].join(':');\n }\n\n private getRawQueryOptions(): GithubHttpOptions {\n const baseUrl = this.baseUrl;\n const repository = `${this.repoOwner}/${this.repoName}`;\n const query = this.datasourceAdapter.query;\n const variables: GithubGraphqlRepoParams = {\n owner: this.repoOwner,\n name: this.repoName,\n count: this.itemsPerQuery,\n cursor: this.cursor,\n };\n\n return {\n baseUrl,\n repository,\n readOnly: true,\n body: { query, variables },\n };\n }\n\n private async doRawQuery(): Promise<\n RawQueryResponse<GithubGraphqlPayload<GraphqlItem>>\n > {\n const requestOptions = this.getRawQueryOptions();\n\n type GraphqlData = GithubGraphqlRepoResponse<GraphqlItem>;\n type HttpBody = GithubGraphqlResponse<GraphqlData>;\n let httpRes: HttpResponse<HttpBody>;\n try {\n httpRes = await this.http.postJson<HttpBody>('/graphql', requestOptions);\n } catch (err) {\n return [null, err];\n }\n\n const { body } = httpRes;\n const { data, errors } = body;\n\n if (errors?.length) {\n if (errors.length === 1) {\n const { message } = errors[0];\n const err = new Error(message);\n return [null, err];\n }\n const errorInstances = errors.map(({ message }) => new Error(message));\n const err = new AggregateError(errorInstances);\n return [null, err];\n }\n\n if (!data) {\n const msg = 'GitHub GraphQL datasource: failed to obtain data';\n const err = new Error(msg);\n return [null, err];\n }\n\n if (!data.repository) {\n const msg = 'GitHub GraphQL datasource: failed to obtain repository data';\n const err = new Error(msg);\n return [null, err];\n }\n\n if (!data.repository.payload) {\n const msg =\n 'GitHub GraphQL datasource: failed to obtain repository payload data';\n const err = new Error(msg);\n return [null, err];\n }\n\n this.queryCount += 1;\n\n // For values other than explicit `false`,\n // we assume that items can not be cached.\n this.isPersistent ??= data.repository.isRepoPrivate === false;\n\n const res = data.repository.payload;\n return [res, null];\n }\n\n private shrinkPageSize(): boolean {\n const nextIdx = pageSizes.indexOf(this.itemsPerQuery) + 1;\n if (nextIdx >= pageSizes.length) {\n return false;\n }\n\n this.itemsPerQuery = pageSizes[nextIdx];\n return true;\n }\n\n private hasReachedQueryLimit(): boolean {\n return this.queryCount >= 100;\n }\n\n private async doShrinkableQuery(): Promise<\n GithubGraphqlPayload<GraphqlItem>\n > {\n let res: GithubGraphqlPayload<GraphqlItem> | null = null;\n let err: Error | null = null;\n\n while (!res) {\n [res, err] = await this.doRawQuery();\n if (err) {\n if (!canBeSolvedByShrinking(err)) {\n throw err;\n }\n\n const shrinkResult = this.shrinkPageSize();\n if (!shrinkResult) {\n throw err;\n }\n const { body: _, ...options } = this.getRawQueryOptions();\n logger.debug(\n { options, newSize: this.itemsPerQuery },\n 'Shrinking GitHub GraphQL page size after error',\n );\n }\n }\n\n return res;\n }\n\n private _cacheStrategy: GithubGraphqlCacheStrategy<ResultItem> | undefined;\n\n private cacheStrategy(): GithubGraphqlCacheStrategy<ResultItem> {\n if (this._cacheStrategy) {\n return this._cacheStrategy;\n }\n const cacheNs = this.getCacheNs();\n const cacheKey = this.getCacheKey();\n const cachePrivatePackages = GlobalConfig.get('cachePrivatePackages');\n const skipStabilization = !is.undefined(this.datasourceAdapter.maxItems);\n this._cacheStrategy =\n cachePrivatePackages || this.isPersistent\n ? new GithubGraphqlPackageCacheStrategy<ResultItem>(\n cacheNs,\n cacheKey,\n skipStabilization,\n )\n : new GithubGraphqlMemoryCacheStrategy<ResultItem>(\n cacheNs,\n cacheKey,\n skipStabilization,\n );\n return this._cacheStrategy;\n }\n\n /**\n * This method is responsible for data synchronization.\n * It also detects persistence of the package, based on the first page result.\n */\n private async doPaginatedFetch(): Promise<void> {\n let hasNextPage = true;\n let isPaginationDone = false;\n let nextCursor: string | undefined;\n let itemCount = 0;\n const maxItems = this.datasourceAdapter.maxItems;\n while (hasNextPage && !isPaginationDone && !this.hasReachedQueryLimit()) {\n if (!is.undefined(maxItems) && itemCount >= maxItems) {\n break;\n }\n const queryResult = await this.doShrinkableQuery();\n\n const resultItems: ResultItem[] = [];\n for (const node of queryResult.nodes) {\n const item = this.datasourceAdapter.transform(node);\n if (!item) {\n logger.once.info(\n {\n packageName: `${this.repoOwner}/${this.repoName}`,\n baseUrl: this.baseUrl,\n },\n `GitHub GraphQL datasource: skipping empty item`,\n );\n continue;\n }\n resultItems.push(item);\n }\n itemCount += resultItems.length;\n\n // It's important to call `getCacheStrategy()` after `doShrinkableQuery()`\n // because `doShrinkableQuery()` may change `this.isCacheable`.\n //\n // Otherwise, cache items for public packages will never be persisted\n // in long-term cache.\n isPaginationDone = await this.cacheStrategy().reconcile(resultItems);\n\n hasNextPage = !!queryResult?.pageInfo?.hasNextPage;\n nextCursor = queryResult?.pageInfo?.endCursor;\n if (hasNextPage && nextCursor) {\n this.cursor = nextCursor;\n }\n }\n\n if (this.isPersistent) {\n await this.storePersistenceFlag(30);\n }\n }\n\n private async doCachedQuery(): Promise<ResultItem[]> {\n await this.loadPersistenceFlag();\n if (!this.isPersistent) {\n await this.doPaginatedFetch();\n }\n\n const res = await this.cacheStrategy().finalizeAndReturn();\n if (res.length) {\n return res;\n }\n\n delete this.isPersistent;\n await this.doPaginatedFetch();\n return this.cacheStrategy().finalizeAndReturn();\n }\n\n async loadPersistenceFlag(): Promise<void> {\n const ns = this.getCacheNs();\n const key = `${this.getCacheKey()}:is-persistent`;\n this.isPersistent = await packageCache.get<true>(ns, key);\n }\n\n async storePersistenceFlag(minutes: number): Promise<void> {\n const ns = this.getCacheNs();\n const key = `${this.getCacheKey()}:is-persistent`;\n await packageCache.set(ns, key, true, minutes);\n }\n\n /**\n * This method ensures the only one query is executed\n * to a particular package during single run.\n */\n private doUniqueQuery(): Promise<ResultItem[]> {\n const cacheKey = `github-pending:${this.getCacheNs()}:${this.getCacheKey()}`;\n let resultPromise = memCache.get<Promise<ResultItem[]>>(cacheKey);\n resultPromise ??= this.doCachedQuery();\n memCache.set(cacheKey, resultPromise);\n return resultPromise;\n }\n\n async getItems(): Promise<ResultItem[]> {\n const res = await this.doUniqueQuery();\n return res;\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;AAoCA,MAAM,YAAY;CAAC;CAAK;CAAI;CAAI;CAAI;AAAC;;;;;;;AAUrC,SAAS,sBAAsB,KAAqB;CAClD,MAAM,EAAE,YAAY;CACpB,OAAO,QAAQ,WAAW,kDAAkD;AAC9E;AAEA,SAAS,uBAAuB,KAAqB;CAEnD,QADwB,eAAe,iBAAiB,IAAI,SAAS,CAAC,GAAG,EAAA,CAC3D,MACX,MAAM,eAAe,qBAAqB,sBAAsB,CAAC,CACpE;AACF;AAEA,IAAa,iCAAb,MAAa,+BAGX;CACA,aAAa,MACX,QACA,MACA,SACc;EAOd,OAAO,MADa,IALC,+BACnB,QACA,MACA,OAEyB,CAAC,CAAC,SAAS;CAExC;CAEA;CACA;CACA;CAEA,gBAAkC,UAAU;CAE5C,aAAqB;CAErB,SAAgC;CAEhC;CACA;CACA;CAKA,YACE,eACA,MACA,mBACA;EACA,KAAK,OAAO;EACZ,KAAK,oBAAoB;EACzB,MAAM,EAAE,aAAa,gBAAgB;EACrC,CAAC,KAAK,WAAW,KAAK,YAAY,YAAY,MAAM,GAAG;EACvD,KAAK,UAAU,cAAc,WAAW,CAAC,CAAC,QAAQ,MAAM,SAAS,GAAG,GAAG;CACzE;CAEA,aAA4C;EAC1C,OAAO,KAAK,kBAAkB;CAChC;CAEA,cAA8B;EAC5B,OAAO;GAAC,KAAK;GAAS,KAAK;GAAW,KAAK;EAAQ,CAAC,CAAC,KAAK,GAAG;CAC/D;CAEA,qBAAgD;EAW9C,OAAO;GACL,SAXc,KAAK;GAYnB,YAAA,GAXoB,KAAK,UAAU,GAAG,KAAK;GAY3C,UAAU;GACV,MAAM;IAAE,OAZI,KAAK,kBAAkB;IAYpB,WAAA;KAVf,OAAO,KAAK;KACZ,MAAM,KAAK;KACX,OAAO,KAAK;KACZ,QAAQ,KAAK;IAOU;GAAE;EAC3B;CACF;CAEA,MAAc,aAEZ;EACA,MAAM,iBAAiB,KAAK,mBAAmB;EAI/C,IAAI;EACJ,IAAI;GACF,UAAU,MAAM,KAAK,KAAK,SAAmB,YAAY,cAAc;EACzE,SAAS,KAAK;GACZ,OAAO,CAAC,MAAM,GAAG;EACnB;EAEA,MAAM,EAAE,SAAS;EACjB,MAAM,EAAE,MAAM,WAAW;EAEzB,IAAI,QAAQ,QAAQ;GAClB,IAAI,OAAO,WAAW,GAAG;IACvB,MAAM,EAAE,YAAY,OAAO;IAE3B,OAAO,CAAC,MAAM,IADE,MAAM,OACN,CAAC;GACnB;GACA,MAAM,iBAAiB,OAAO,KAAK,EAAE,cAAc,IAAI,MAAM,OAAO,CAAC;GAErE,OAAO,CAAC,MAAM,IADE,eAAe,cACf,CAAC;EACnB;EAEA,IAAI,CAAC,MAGH,OAAO,CAAC,sBAAM,IADE,MAAM,kDACN,CAAC;EAGnB,IAAI,CAAC,KAAK,YAGR,OAAO,CAAC,sBAAM,IADE,MAAM,6DACN,CAAC;EAGnB,IAAI,CAAC,KAAK,WAAW,SAInB,OAAO,CAAC,sBAAM,IADE,MAAM,qEACN,CAAC;EAGnB,KAAK,cAAc;EAInB,KAAK,iBAAiB,KAAK,WAAW,kBAAkB;EAGxD,OAAO,CADK,KAAK,WAAW,SACf,IAAI;CACnB;CAEA,iBAAkC;EAChC,MAAM,UAAU,UAAU,QAAQ,KAAK,aAAa,IAAI;EACxD,IAAI,WAAW,UAAU,QACvB,OAAO;EAGT,KAAK,gBAAgB,UAAU;EAC/B,OAAO;CACT;CAEA,uBAAwC;EACtC,OAAO,KAAK,cAAc;CAC5B;CAEA,MAAc,oBAEZ;EACA,IAAI,MAAgD;EACpD,IAAI,MAAoB;EAExB,OAAO,CAAC,KAAK;GACX,CAAC,KAAK,OAAO,MAAM,KAAK,WAAW;GACnC,IAAI,KAAK;IACP,IAAI,CAAC,uBAAuB,GAAG,GAC7B,MAAM;IAIR,IAAI,CADiB,KAAK,eACV,GACd,MAAM;IAER,MAAM,EAAE,MAAM,GAAG,GAAG,YAAY,KAAK,mBAAmB;IACxD,OAAO,MACL;KAAE;KAAS,SAAS,KAAK;IAAc,GACvC,gDACF;GACF;EACF;EAEA,OAAO;CACT;CAEA;CAEA,gBAAgE;EAC9D,IAAI,KAAK,gBACP,OAAO,KAAK;EAEd,MAAM,UAAU,KAAK,WAAW;EAChC,MAAM,WAAW,KAAK,YAAY;EAClC,MAAM,uBAAuB,aAAa,IAAI,sBAAsB;EACpE,MAAM,oBAAoB,CAAC,GAAG,UAAU,KAAK,kBAAkB,QAAQ;EACvE,KAAK,iBACH,wBAAwB,KAAK,eACzB,IAAI,kCACF,SACA,UACA,iBACF,IACA,IAAI,iCACF,SACA,UACA,iBACF;EACN,OAAO,KAAK;CACd;;;;;CAMA,MAAc,mBAAkC;EAC9C,IAAI,cAAc;EAClB,IAAI,mBAAmB;EACvB,IAAI;EACJ,IAAI,YAAY;EAChB,MAAM,WAAW,KAAK,kBAAkB;EACxC,OAAO,eAAe,CAAC,oBAAoB,CAAC,KAAK,qBAAqB,GAAG;GACvE,IAAI,CAAC,GAAG,UAAU,QAAQ,KAAK,aAAa,UAC1C;GAEF,MAAM,cAAc,MAAM,KAAK,kBAAkB;GAEjD,MAAM,cAA4B,CAAC;GACnC,KAAK,MAAM,QAAQ,YAAY,OAAO;IACpC,MAAM,OAAO,KAAK,kBAAkB,UAAU,IAAI;IAClD,IAAI,CAAC,MAAM;KACT,OAAO,KAAK,KACV;MACE,aAAa,GAAG,KAAK,UAAU,GAAG,KAAK;MACvC,SAAS,KAAK;KAChB,GACA,gDACF;KACA;IACF;IACA,YAAY,KAAK,IAAI;GACvB;GACA,aAAa,YAAY;GAOzB,mBAAmB,MAAM,KAAK,cAAc,CAAC,CAAC,UAAU,WAAW;GAEnE,cAAc,CAAC,CAAC,aAAa,UAAU;GACvC,aAAa,aAAa,UAAU;GACpC,IAAI,eAAe,YACjB,KAAK,SAAS;EAElB;EAEA,IAAI,KAAK,cACP,MAAM,KAAK,qBAAqB,EAAE;CAEtC;CAEA,MAAc,gBAAuC;EACnD,MAAM,KAAK,oBAAoB;EAC/B,IAAI,CAAC,KAAK,cACR,MAAM,KAAK,iBAAiB;EAG9B,MAAM,MAAM,MAAM,KAAK,cAAc,CAAC,CAAC,kBAAkB;EACzD,IAAI,IAAI,QACN,OAAO;EAGT,OAAO,KAAK;EACZ,MAAM,KAAK,iBAAiB;EAC5B,OAAO,KAAK,cAAc,CAAC,CAAC,kBAAkB;CAChD;CAEA,MAAM,sBAAqC;EACzC,MAAM,KAAK,KAAK,WAAW;EAC3B,MAAM,MAAM,GAAG,KAAK,YAAY,EAAE;EAClC,KAAK,eAAe,MAAMA,MAAuB,IAAI,GAAG;CAC1D;CAEA,MAAM,qBAAqB,SAAgC;EACzD,MAAM,KAAK,KAAK,WAAW;EAC3B,MAAM,MAAM,GAAG,KAAK,YAAY,EAAE;EAClC,MAAMC,MAAiB,IAAI,KAAK,MAAM,OAAO;CAC/C;;;;;CAMA,gBAA+C;EAC7C,MAAM,WAAW,kBAAkB,KAAK,WAAW,EAAE,GAAG,KAAK,YAAY;EACzE,IAAI,gBAAgBC,IAAoC,QAAQ;EAChE,kBAAkB,KAAK,cAAc;EACrC,IAAa,UAAU,aAAa;EACpC,OAAO;CACT;CAEA,MAAM,WAAkC;EAEtC,OAAO,MADW,KAAK,cAAc;CAEvC;AACF"}
|
package/package.json
CHANGED
package/renovate-schema.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$id": "https://docs.renovatebot.com/renovate-schema.json",
|
|
3
|
-
"title": "JSON schema for Renovate 44.45.
|
|
3
|
+
"title": "JSON schema for Renovate 44.45.2 config files (https://renovatebot.com/)",
|
|
4
4
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
5
|
-
"x-renovate-version": "44.45.
|
|
5
|
+
"x-renovate-version": "44.45.2",
|
|
6
6
|
"allowComments": true,
|
|
7
7
|
"type": "object",
|
|
8
8
|
"definitions": {
|