utilium 1.2.3 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +58 -0
- package/dist/cache.js +121 -0
- package/dist/debugging.js +29 -6
- package/dist/requests.d.ts +14 -65
- package/dist/requests.js +14 -130
- package/package.json +1 -1
- package/readme.md +1 -0
- package/src/cache.ts +169 -0
- package/src/debugging.ts +28 -5
- package/src/requests.ts +26 -184
package/dist/cache.d.ts
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
export interface Options {
|
2
|
+
/**
|
3
|
+
* If true, use multiple buffers to cache a file.
|
4
|
+
* This is useful when working with small parts of large files,
|
5
|
+
* since we don't need to allocate a large buffer that is mostly unused
|
6
|
+
* @default true
|
7
|
+
*/
|
8
|
+
sparse?: boolean;
|
9
|
+
/**
|
10
|
+
* The threshold for whether to combine regions or not
|
11
|
+
* @see Region
|
12
|
+
* @default 0xfff // 4 KiB
|
13
|
+
*/
|
14
|
+
regionGapThreshold?: number;
|
15
|
+
/**
|
16
|
+
* Whether to only update the cache when changing or deleting resources
|
17
|
+
* @default false
|
18
|
+
*/
|
19
|
+
cacheOnly?: boolean;
|
20
|
+
}
|
21
|
+
export type Range = {
|
22
|
+
start: number;
|
23
|
+
end: number;
|
24
|
+
};
|
25
|
+
export interface Region {
|
26
|
+
/** The region's offset from the start of the resource */
|
27
|
+
offset: number;
|
28
|
+
/** Ranges cached in this region. These are absolute! */
|
29
|
+
ranges: Range[];
|
30
|
+
/** Data for this region */
|
31
|
+
data: Uint8Array;
|
32
|
+
}
|
33
|
+
/**
|
34
|
+
* The cache for a specific resource
|
35
|
+
* @internal
|
36
|
+
*/
|
37
|
+
export declare class Resource {
|
38
|
+
/** The resource ID */
|
39
|
+
readonly id: string;
|
40
|
+
/** The full size of the resource */
|
41
|
+
readonly size: number;
|
42
|
+
protected readonly options: Options;
|
43
|
+
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
44
|
+
readonly regions: Region[];
|
45
|
+
constructor(
|
46
|
+
/** The resource ID */
|
47
|
+
id: string,
|
48
|
+
/** The full size of the resource */
|
49
|
+
size: number, options: Options, resources?: Map<string, Resource | undefined>);
|
50
|
+
/** Combines adjacent regions and combines adjacent ranges within a region */
|
51
|
+
collect(): void;
|
52
|
+
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
53
|
+
missing(start: number, end: number): Range[];
|
54
|
+
/** Get the region who's ranges include an offset */
|
55
|
+
regionAt(offset: number): Region | undefined;
|
56
|
+
/** Add new data to the cache at given specified offset */
|
57
|
+
add(data: Uint8Array, offset: number): this;
|
58
|
+
}
|
package/dist/cache.js
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
/** A ranged cache */
|
2
|
+
import { extendBuffer } from './buffer.js';
|
3
|
+
/**
|
4
|
+
* The cache for a specific resource
|
5
|
+
* @internal
|
6
|
+
*/
|
7
|
+
export class Resource {
|
8
|
+
id;
|
9
|
+
size;
|
10
|
+
options;
|
11
|
+
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
12
|
+
regions = [];
|
13
|
+
constructor(
|
14
|
+
/** The resource ID */
|
15
|
+
id,
|
16
|
+
/** The full size of the resource */
|
17
|
+
size, options, resources) {
|
18
|
+
this.id = id;
|
19
|
+
this.size = size;
|
20
|
+
this.options = options;
|
21
|
+
options.sparse ??= true;
|
22
|
+
if (!options.sparse)
|
23
|
+
this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
24
|
+
resources?.set(id, this);
|
25
|
+
}
|
26
|
+
/** Combines adjacent regions and combines adjacent ranges within a region */
|
27
|
+
collect() {
|
28
|
+
if (!this.options.sparse)
|
29
|
+
return;
|
30
|
+
const { regionGapThreshold = 0xfff } = this.options;
|
31
|
+
for (let i = 0; i < this.regions.length - 1;) {
|
32
|
+
const current = this.regions[i];
|
33
|
+
const next = this.regions[i + 1];
|
34
|
+
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
35
|
+
i++;
|
36
|
+
continue;
|
37
|
+
}
|
38
|
+
// Combine ranges
|
39
|
+
current.ranges.push(...next.ranges);
|
40
|
+
current.ranges.sort((a, b) => a.start - b.start);
|
41
|
+
// Combine overlapping/adjacent ranges
|
42
|
+
current.ranges = current.ranges.reduce((acc, range) => {
|
43
|
+
if (!acc.length || acc.at(-1).end < range.start) {
|
44
|
+
acc.push(range);
|
45
|
+
}
|
46
|
+
else {
|
47
|
+
acc.at(-1).end = Math.max(acc.at(-1).end, range.end);
|
48
|
+
}
|
49
|
+
return acc;
|
50
|
+
}, []);
|
51
|
+
// Extend buffer to include the new region
|
52
|
+
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
53
|
+
current.data.set(next.data, next.offset - current.offset);
|
54
|
+
// Remove the next region after merging
|
55
|
+
this.regions.splice(i + 1, 1);
|
56
|
+
}
|
57
|
+
}
|
58
|
+
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
59
|
+
missing(start, end) {
|
60
|
+
const missingRanges = [];
|
61
|
+
for (const region of this.regions) {
|
62
|
+
if (region.offset >= end)
|
63
|
+
break;
|
64
|
+
for (const range of region.ranges) {
|
65
|
+
if (range.end <= start)
|
66
|
+
continue;
|
67
|
+
if (range.start >= end)
|
68
|
+
break;
|
69
|
+
if (range.start > start) {
|
70
|
+
missingRanges.push({ start, end: Math.min(range.start, end) });
|
71
|
+
}
|
72
|
+
// Adjust the current start if the region overlaps
|
73
|
+
if (range.end > start)
|
74
|
+
start = Math.max(start, range.end);
|
75
|
+
if (start >= end)
|
76
|
+
break;
|
77
|
+
}
|
78
|
+
if (start >= end)
|
79
|
+
break;
|
80
|
+
}
|
81
|
+
// If there are still missing parts at the end
|
82
|
+
if (start < end)
|
83
|
+
missingRanges.push({ start, end });
|
84
|
+
return missingRanges;
|
85
|
+
}
|
86
|
+
/** Get the region who's ranges include an offset */
|
87
|
+
regionAt(offset) {
|
88
|
+
if (!this.regions.length)
|
89
|
+
return;
|
90
|
+
for (const region of this.regions) {
|
91
|
+
if (region.offset > offset)
|
92
|
+
break;
|
93
|
+
// Check if the offset is within this region
|
94
|
+
if (offset >= region.offset && offset < region.offset + region.data.byteLength)
|
95
|
+
return region;
|
96
|
+
}
|
97
|
+
}
|
98
|
+
/** Add new data to the cache at given specified offset */
|
99
|
+
add(data, offset) {
|
100
|
+
const end = offset + data.byteLength;
|
101
|
+
const region = this.regionAt(offset);
|
102
|
+
if (region) {
|
103
|
+
region.data = extendBuffer(region.data, end);
|
104
|
+
region.data.set(data, offset);
|
105
|
+
region.ranges.push({ start: offset, end });
|
106
|
+
region.ranges.sort((a, b) => a.start - b.start);
|
107
|
+
return this;
|
108
|
+
}
|
109
|
+
// Find the correct index to insert the new region
|
110
|
+
const newRegion = { data, offset: offset, ranges: [{ start: offset, end }] };
|
111
|
+
const insertIndex = this.regions.findIndex(region => region.offset > offset);
|
112
|
+
// Insert at the right index to keep regions sorted
|
113
|
+
if (insertIndex == -1) {
|
114
|
+
this.regions.push(newRegion); // Append if no later region exists
|
115
|
+
}
|
116
|
+
else {
|
117
|
+
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
118
|
+
}
|
119
|
+
return this;
|
120
|
+
}
|
121
|
+
}
|
package/dist/debugging.js
CHANGED
@@ -1,10 +1,33 @@
|
|
1
1
|
function defaultStringify(value) {
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
2
|
+
switch (typeof value) {
|
3
|
+
case 'undefined':
|
4
|
+
return 'undefined';
|
5
|
+
case 'string':
|
6
|
+
case 'number':
|
7
|
+
case 'boolean':
|
8
|
+
return JSON.stringify(value);
|
9
|
+
case 'bigint':
|
10
|
+
case 'symbol':
|
11
|
+
return value.toString();
|
12
|
+
case 'function':
|
13
|
+
return value.name + '()';
|
14
|
+
case 'object':
|
15
|
+
if (value === null)
|
16
|
+
return 'null';
|
17
|
+
if (ArrayBuffer.isView(value)) {
|
18
|
+
const length = 'length' in value ? value.length : value.byteLength / value.constructor.BYTES_PER_ELEMENT;
|
19
|
+
return `${value.constructor.name.replaceAll('Array', '').toLowerCase()}[${length}]`;
|
20
|
+
}
|
21
|
+
if (Array.isArray(value))
|
22
|
+
return `unknown[${value.length}]`;
|
23
|
+
try {
|
24
|
+
const json = JSON.stringify(value);
|
25
|
+
return json.length < 100 ? json : value.toString();
|
26
|
+
}
|
27
|
+
catch {
|
28
|
+
return value.toString();
|
29
|
+
}
|
30
|
+
}
|
8
31
|
}
|
9
32
|
/**
|
10
33
|
* Create a function that can be used to decorate classes and non-field members.
|
package/dist/requests.d.ts
CHANGED
@@ -1,69 +1,18 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
* @default 0xfff // 4 KiB
|
13
|
-
*/
|
14
|
-
regionGapThreshold?: number;
|
15
|
-
/**
|
16
|
-
* Whether to only update the cache when changing or deleting resources
|
17
|
-
* @default false
|
18
|
-
*/
|
19
|
-
cacheOnly?: boolean;
|
20
|
-
}
|
21
|
-
/**
|
22
|
-
* @deprecated Use `CacheOptions`
|
23
|
-
*/
|
24
|
-
export type ResourceCacheOptions = CacheOptions;
|
25
|
-
export type CacheRange = {
|
26
|
-
start: number;
|
27
|
-
end: number;
|
28
|
-
};
|
29
|
-
export interface CacheRegion {
|
30
|
-
/** The region's offset from the start of the resource */
|
31
|
-
offset: number;
|
32
|
-
/** Ranges cached in this region. These are absolute! */
|
33
|
-
ranges: CacheRange[];
|
34
|
-
/** Data for this region */
|
35
|
-
data: Uint8Array;
|
36
|
-
}
|
37
|
-
/**
|
38
|
-
* The cache for a specific resource
|
39
|
-
* @internal
|
40
|
-
*/
|
41
|
-
export declare class ResourceCache {
|
42
|
-
/** The resource URL */
|
43
|
-
readonly url: string;
|
44
|
-
/** The full size of the resource */
|
45
|
-
readonly size: number;
|
46
|
-
protected readonly options: CacheOptions;
|
47
|
-
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
48
|
-
readonly regions: CacheRegion[];
|
49
|
-
constructor(
|
50
|
-
/** The resource URL */
|
51
|
-
url: string,
|
52
|
-
/** The full size of the resource */
|
53
|
-
size: number, options: CacheOptions);
|
54
|
-
/** Combines adjacent regions and combines adjacent ranges within a region */
|
55
|
-
collect(): void;
|
56
|
-
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
57
|
-
missing(start: number, end: number): CacheRange[];
|
58
|
-
/** Get the region who's ranges include an offset */
|
59
|
-
regionAt(offset: number): CacheRegion | undefined;
|
60
|
-
/** Add new data to the cache at given specified offset */
|
61
|
-
add(data: Uint8Array, offset: number): this;
|
62
|
-
}
|
1
|
+
import * as cache from './cache.js';
|
2
|
+
/** @deprecated Use `cache.Options` */
|
3
|
+
export type ResourceCacheOptions = cache.Options;
|
4
|
+
/** @deprecated Use `cache.Resource` */
|
5
|
+
export declare const ResourceCache: typeof cache.Resource;
|
6
|
+
/** @deprecated Use `cache.Resource` */
|
7
|
+
export type ResourceCache = cache.Resource;
|
8
|
+
/** @deprecated Use `cache.Range` */
|
9
|
+
export type CacheRange = cache.Range;
|
10
|
+
/** @deprecated Use `cache.Options` */
|
11
|
+
export type CacheOptions = cache.Options;
|
63
12
|
/**
|
64
13
|
* @internal
|
65
14
|
*/
|
66
|
-
export declare const resourcesCache: Map<string,
|
15
|
+
export declare const resourcesCache: Map<string, cache.Resource | undefined>;
|
67
16
|
export type Issue = {
|
68
17
|
tag: 'status';
|
69
18
|
response: Response;
|
@@ -79,7 +28,7 @@ export type Issue = {
|
|
79
28
|
* @deprecated Use `Issue`
|
80
29
|
*/
|
81
30
|
export type RequestError = Issue;
|
82
|
-
export interface Options extends
|
31
|
+
export interface Options extends cache.Options {
|
83
32
|
/** Optionally provide a function for logging warnings */
|
84
33
|
warn?(message: string): unknown;
|
85
34
|
}
|
@@ -114,7 +63,7 @@ export declare const GET: typeof get;
|
|
114
63
|
*/
|
115
64
|
export declare function getCached(url: string, options: GetOptions): {
|
116
65
|
data?: Uint8Array;
|
117
|
-
missing:
|
66
|
+
missing: cache.Range[];
|
118
67
|
};
|
119
68
|
interface SetOptions extends Options {
|
120
69
|
/** The offset we are updating at */
|
package/dist/requests.js
CHANGED
@@ -1,124 +1,8 @@
|
|
1
1
|
/* Utilities for `fetch` when using range requests. It also allows you to handle errors easier */
|
2
|
-
import
|
3
|
-
/**
|
4
|
-
|
5
|
-
|
6
|
-
*/
|
7
|
-
export class ResourceCache {
|
8
|
-
url;
|
9
|
-
size;
|
10
|
-
options;
|
11
|
-
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
12
|
-
regions = [];
|
13
|
-
constructor(
|
14
|
-
/** The resource URL */
|
15
|
-
url,
|
16
|
-
/** The full size of the resource */
|
17
|
-
size, options) {
|
18
|
-
this.url = url;
|
19
|
-
this.size = size;
|
20
|
-
this.options = options;
|
21
|
-
options.sparse ??= true;
|
22
|
-
if (!options.sparse)
|
23
|
-
this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
24
|
-
resourcesCache.set(url, this);
|
25
|
-
}
|
26
|
-
/** Combines adjacent regions and combines adjacent ranges within a region */
|
27
|
-
collect() {
|
28
|
-
if (!this.options.sparse)
|
29
|
-
return;
|
30
|
-
const { regionGapThreshold = 0xfff } = this.options;
|
31
|
-
for (let i = 0; i < this.regions.length - 1;) {
|
32
|
-
const current = this.regions[i];
|
33
|
-
const next = this.regions[i + 1];
|
34
|
-
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
35
|
-
i++;
|
36
|
-
continue;
|
37
|
-
}
|
38
|
-
// Combine ranges
|
39
|
-
current.ranges.push(...next.ranges);
|
40
|
-
current.ranges.sort((a, b) => a.start - b.start);
|
41
|
-
// Combine overlapping/adjacent ranges
|
42
|
-
current.ranges = current.ranges.reduce((acc, range) => {
|
43
|
-
if (!acc.length || acc.at(-1).end < range.start) {
|
44
|
-
acc.push(range);
|
45
|
-
}
|
46
|
-
else {
|
47
|
-
acc.at(-1).end = Math.max(acc.at(-1).end, range.end);
|
48
|
-
}
|
49
|
-
return acc;
|
50
|
-
}, []);
|
51
|
-
// Extend buffer to include the new region
|
52
|
-
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
53
|
-
current.data.set(next.data, next.offset - current.offset);
|
54
|
-
// Remove the next region after merging
|
55
|
-
this.regions.splice(i + 1, 1);
|
56
|
-
}
|
57
|
-
}
|
58
|
-
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
59
|
-
missing(start, end) {
|
60
|
-
const missingRanges = [];
|
61
|
-
for (const region of this.regions) {
|
62
|
-
if (region.offset >= end)
|
63
|
-
break;
|
64
|
-
for (const range of region.ranges) {
|
65
|
-
if (range.end <= start)
|
66
|
-
continue;
|
67
|
-
if (range.start >= end)
|
68
|
-
break;
|
69
|
-
if (range.start > start) {
|
70
|
-
missingRanges.push({ start, end: Math.min(range.start, end) });
|
71
|
-
}
|
72
|
-
// Adjust the current start if the region overlaps
|
73
|
-
if (range.end > start)
|
74
|
-
start = Math.max(start, range.end);
|
75
|
-
if (start >= end)
|
76
|
-
break;
|
77
|
-
}
|
78
|
-
if (start >= end)
|
79
|
-
break;
|
80
|
-
}
|
81
|
-
// If there are still missing parts at the end
|
82
|
-
if (start < end)
|
83
|
-
missingRanges.push({ start, end });
|
84
|
-
return missingRanges;
|
85
|
-
}
|
86
|
-
/** Get the region who's ranges include an offset */
|
87
|
-
regionAt(offset) {
|
88
|
-
if (!this.regions.length)
|
89
|
-
return;
|
90
|
-
for (const region of this.regions) {
|
91
|
-
if (region.offset > offset)
|
92
|
-
break;
|
93
|
-
// Check if the offset is within this region
|
94
|
-
if (offset >= region.offset && offset < region.offset + region.data.byteLength)
|
95
|
-
return region;
|
96
|
-
}
|
97
|
-
}
|
98
|
-
/** Add new data to the cache at given specified offset */
|
99
|
-
add(data, offset) {
|
100
|
-
const end = offset + data.byteLength;
|
101
|
-
const region = this.regionAt(offset);
|
102
|
-
if (region) {
|
103
|
-
region.data = extendBuffer(region.data, end);
|
104
|
-
region.data.set(data, offset);
|
105
|
-
region.ranges.push({ start: offset, end });
|
106
|
-
region.ranges.sort((a, b) => a.start - b.start);
|
107
|
-
return this;
|
108
|
-
}
|
109
|
-
// Find the correct index to insert the new region
|
110
|
-
const newRegion = { data, offset: offset, ranges: [{ start: offset, end }] };
|
111
|
-
const insertIndex = this.regions.findIndex(region => region.offset > offset);
|
112
|
-
// Insert at the right index to keep regions sorted
|
113
|
-
if (insertIndex == -1) {
|
114
|
-
this.regions.push(newRegion); // Append if no later region exists
|
115
|
-
}
|
116
|
-
else {
|
117
|
-
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
118
|
-
}
|
119
|
-
return this;
|
120
|
-
}
|
121
|
-
}
|
2
|
+
import * as cache from './cache.js';
|
3
|
+
/** @deprecated Use `cache.Resource` */
|
4
|
+
export const ResourceCache = cache.Resource;
|
5
|
+
/* eslint-disable @typescript-eslint/only-throw-error */
|
122
6
|
/**
|
123
7
|
* @internal
|
124
8
|
*/
|
@@ -149,7 +33,7 @@ export async function get(url, options, init = {}) {
|
|
149
33
|
// Request no using ranges
|
150
34
|
if (typeof options.start != 'number' || typeof options.end != 'number') {
|
151
35
|
const { data } = await _fetch(url, init);
|
152
|
-
new
|
36
|
+
new cache.Resource(url, data.byteLength, options, resourcesCache).add(data, 0);
|
153
37
|
return data;
|
154
38
|
}
|
155
39
|
// Range requests
|
@@ -162,22 +46,22 @@ export async function get(url, options, init = {}) {
|
|
162
46
|
options.size = size;
|
163
47
|
}
|
164
48
|
const { size, start, end } = options;
|
165
|
-
const
|
49
|
+
const resource = resourcesCache.get(url) ?? new cache.Resource(url, size, options, resourcesCache);
|
166
50
|
req.headers.set('If-Range', new Date().toUTCString());
|
167
|
-
for (const { start: from, end: to } of
|
51
|
+
for (const { start: from, end: to } of resource.missing(start, end)) {
|
168
52
|
const { data, response } = await _fetch(req, { headers: { Range: `bytes=${from}-${to}` } });
|
169
53
|
if (response.status == 206) {
|
170
|
-
|
54
|
+
resource.add(data, from);
|
171
55
|
continue;
|
172
56
|
}
|
173
57
|
// The first response doesn't have a "partial content" (206) status
|
174
58
|
options.warn?.(url + ': Remote does not support range requests with bytes. Falling back to full data.');
|
175
|
-
new
|
59
|
+
new cache.Resource(url, size, options, resourcesCache).add(data, 0);
|
176
60
|
return data.subarray(start, end);
|
177
61
|
}
|
178
62
|
// This ensures we get a single buffer with the entire requested range
|
179
|
-
|
180
|
-
const region =
|
63
|
+
resource.collect();
|
64
|
+
const region = resource.regionAt(start);
|
181
65
|
return region.data.subarray(start - region.offset, end - region.offset);
|
182
66
|
}
|
183
67
|
/**
|
@@ -226,13 +110,13 @@ export function getCached(url, options) {
|
|
226
110
|
*/
|
227
111
|
export async function set(url, data, options, init = {}) {
|
228
112
|
if (!resourcesCache.has(url)) {
|
229
|
-
new
|
113
|
+
new cache.Resource(url, options.size ?? data.byteLength, options, resourcesCache);
|
230
114
|
}
|
231
|
-
const
|
115
|
+
const resource = resourcesCache.get(url);
|
232
116
|
const { offset = 0 } = options;
|
233
117
|
if (!options.cacheOnly)
|
234
118
|
await _fetch(new Request(url, init), { method: 'POST' }, true);
|
235
|
-
|
119
|
+
resource.add(data, offset).collect();
|
236
120
|
}
|
237
121
|
/**
|
238
122
|
* Make a DELETE request to remove the resource from the server and clear it from the cache.
|
package/package.json
CHANGED
package/readme.md
CHANGED
@@ -9,6 +9,7 @@ A bunch of utilities for Typescript. This includes:
|
|
9
9
|
- Extending buffers easier with `extendBuffer`
|
10
10
|
- Convenience types and functions for strings and objects
|
11
11
|
- RNG functions
|
12
|
+
- Ranged cache
|
12
13
|
- `List`, a class that combines the best aspects of `Set` and arrays
|
13
14
|
- `JSONFileMap` and `FolderMap`
|
14
15
|
- Version utilities
|
package/src/cache.ts
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
/** A ranged cache */
|
2
|
+
import { extendBuffer } from './buffer.js';
|
3
|
+
|
4
|
+
export interface Options {
|
5
|
+
/**
|
6
|
+
* If true, use multiple buffers to cache a file.
|
7
|
+
* This is useful when working with small parts of large files,
|
8
|
+
* since we don't need to allocate a large buffer that is mostly unused
|
9
|
+
* @default true
|
10
|
+
*/
|
11
|
+
sparse?: boolean;
|
12
|
+
|
13
|
+
/**
|
14
|
+
* The threshold for whether to combine regions or not
|
15
|
+
* @see Region
|
16
|
+
* @default 0xfff // 4 KiB
|
17
|
+
*/
|
18
|
+
regionGapThreshold?: number;
|
19
|
+
|
20
|
+
/**
|
21
|
+
* Whether to only update the cache when changing or deleting resources
|
22
|
+
* @default false
|
23
|
+
*/
|
24
|
+
cacheOnly?: boolean;
|
25
|
+
}
|
26
|
+
|
27
|
+
export type Range = { start: number; end: number };
|
28
|
+
|
29
|
+
export interface Region {
|
30
|
+
/** The region's offset from the start of the resource */
|
31
|
+
offset: number;
|
32
|
+
|
33
|
+
/** Ranges cached in this region. These are absolute! */
|
34
|
+
ranges: Range[];
|
35
|
+
|
36
|
+
/** Data for this region */
|
37
|
+
data: Uint8Array;
|
38
|
+
}
|
39
|
+
|
40
|
+
/**
|
41
|
+
* The cache for a specific resource
|
42
|
+
* @internal
|
43
|
+
*/
|
44
|
+
export class Resource {
|
45
|
+
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
46
|
+
public readonly regions: Region[] = [];
|
47
|
+
|
48
|
+
public constructor(
|
49
|
+
/** The resource ID */
|
50
|
+
public readonly id: string,
|
51
|
+
/** The full size of the resource */
|
52
|
+
public readonly size: number,
|
53
|
+
protected readonly options: Options,
|
54
|
+
resources?: Map<string, Resource | undefined>
|
55
|
+
) {
|
56
|
+
options.sparse ??= true;
|
57
|
+
if (!options.sparse) this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
58
|
+
|
59
|
+
resources?.set(id, this);
|
60
|
+
}
|
61
|
+
|
62
|
+
/** Combines adjacent regions and combines adjacent ranges within a region */
|
63
|
+
public collect(): void {
|
64
|
+
if (!this.options.sparse) return;
|
65
|
+
const { regionGapThreshold = 0xfff } = this.options;
|
66
|
+
|
67
|
+
for (let i = 0; i < this.regions.length - 1; ) {
|
68
|
+
const current = this.regions[i];
|
69
|
+
const next = this.regions[i + 1];
|
70
|
+
|
71
|
+
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
72
|
+
i++;
|
73
|
+
continue;
|
74
|
+
}
|
75
|
+
|
76
|
+
// Combine ranges
|
77
|
+
current.ranges.push(...next.ranges);
|
78
|
+
current.ranges.sort((a, b) => a.start - b.start);
|
79
|
+
|
80
|
+
// Combine overlapping/adjacent ranges
|
81
|
+
current.ranges = current.ranges.reduce((acc: Range[], range) => {
|
82
|
+
if (!acc.length || acc.at(-1)!.end < range.start) {
|
83
|
+
acc.push(range);
|
84
|
+
} else {
|
85
|
+
acc.at(-1)!.end = Math.max(acc.at(-1)!.end, range.end);
|
86
|
+
}
|
87
|
+
return acc;
|
88
|
+
}, []);
|
89
|
+
|
90
|
+
// Extend buffer to include the new region
|
91
|
+
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
92
|
+
current.data.set(next.data, next.offset - current.offset);
|
93
|
+
|
94
|
+
// Remove the next region after merging
|
95
|
+
this.regions.splice(i + 1, 1);
|
96
|
+
}
|
97
|
+
}
|
98
|
+
|
99
|
+
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
100
|
+
public missing(start: number, end: number): Range[] {
|
101
|
+
const missingRanges: Range[] = [];
|
102
|
+
|
103
|
+
for (const region of this.regions) {
|
104
|
+
if (region.offset >= end) break;
|
105
|
+
|
106
|
+
for (const range of region.ranges) {
|
107
|
+
if (range.end <= start) continue;
|
108
|
+
|
109
|
+
if (range.start >= end) break;
|
110
|
+
|
111
|
+
if (range.start > start) {
|
112
|
+
missingRanges.push({ start, end: Math.min(range.start, end) });
|
113
|
+
}
|
114
|
+
|
115
|
+
// Adjust the current start if the region overlaps
|
116
|
+
if (range.end > start) start = Math.max(start, range.end);
|
117
|
+
|
118
|
+
if (start >= end) break;
|
119
|
+
}
|
120
|
+
|
121
|
+
if (start >= end) break;
|
122
|
+
}
|
123
|
+
|
124
|
+
// If there are still missing parts at the end
|
125
|
+
if (start < end) missingRanges.push({ start, end });
|
126
|
+
|
127
|
+
return missingRanges;
|
128
|
+
}
|
129
|
+
|
130
|
+
/** Get the region who's ranges include an offset */
|
131
|
+
public regionAt(offset: number): Region | undefined {
|
132
|
+
if (!this.regions.length) return;
|
133
|
+
|
134
|
+
for (const region of this.regions) {
|
135
|
+
if (region.offset > offset) break;
|
136
|
+
|
137
|
+
// Check if the offset is within this region
|
138
|
+
if (offset >= region.offset && offset < region.offset + region.data.byteLength) return region;
|
139
|
+
}
|
140
|
+
}
|
141
|
+
|
142
|
+
/** Add new data to the cache at given specified offset */
|
143
|
+
public add(data: Uint8Array, offset: number): this {
|
144
|
+
const end = offset + data.byteLength;
|
145
|
+
const region = this.regionAt(offset);
|
146
|
+
|
147
|
+
if (region) {
|
148
|
+
region.data = extendBuffer(region.data, end);
|
149
|
+
region.data.set(data, offset);
|
150
|
+
region.ranges.push({ start: offset, end });
|
151
|
+
region.ranges.sort((a, b) => a.start - b.start);
|
152
|
+
|
153
|
+
return this;
|
154
|
+
}
|
155
|
+
|
156
|
+
// Find the correct index to insert the new region
|
157
|
+
const newRegion: Region = { data, offset: offset, ranges: [{ start: offset, end }] };
|
158
|
+
const insertIndex = this.regions.findIndex(region => region.offset > offset);
|
159
|
+
|
160
|
+
// Insert at the right index to keep regions sorted
|
161
|
+
if (insertIndex == -1) {
|
162
|
+
this.regions.push(newRegion); // Append if no later region exists
|
163
|
+
} else {
|
164
|
+
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
165
|
+
}
|
166
|
+
|
167
|
+
return this;
|
168
|
+
}
|
169
|
+
}
|
package/src/debugging.ts
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
/* eslint-disable @typescript-eslint/no-explicit-any */
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-base-to-string */
|
2
2
|
export interface CreateLoggerOptions {
|
3
3
|
/**
|
4
4
|
* The function used to output
|
@@ -25,10 +25,33 @@ export interface CreateLoggerOptions {
|
|
25
25
|
}
|
26
26
|
|
27
27
|
function defaultStringify(value: unknown): string {
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
28
|
+
switch (typeof value) {
|
29
|
+
case 'undefined':
|
30
|
+
return 'undefined';
|
31
|
+
case 'string':
|
32
|
+
case 'number':
|
33
|
+
case 'boolean':
|
34
|
+
return JSON.stringify(value);
|
35
|
+
case 'bigint':
|
36
|
+
case 'symbol':
|
37
|
+
return value.toString();
|
38
|
+
case 'function':
|
39
|
+
return value.name + '()';
|
40
|
+
case 'object':
|
41
|
+
if (value === null) return 'null';
|
42
|
+
if (ArrayBuffer.isView(value)) {
|
43
|
+
const length = 'length' in value ? (value.length as number) : value.byteLength / (value.constructor as any).BYTES_PER_ELEMENT;
|
44
|
+
return `${value.constructor.name.replaceAll('Array', '').toLowerCase()}[${length}]`;
|
45
|
+
}
|
46
|
+
if (Array.isArray(value)) return `unknown[${value.length}]`;
|
47
|
+
try {
|
48
|
+
const json = JSON.stringify(value);
|
49
|
+
|
50
|
+
return json.length < 100 ? json : value.toString();
|
51
|
+
} catch {
|
52
|
+
return value.toString();
|
53
|
+
}
|
54
|
+
}
|
32
55
|
}
|
33
56
|
|
34
57
|
type LoggableDecoratorContext = Exclude<DecoratorContext, ClassFieldDecoratorContext>;
|
package/src/requests.ts
CHANGED
@@ -1,184 +1,26 @@
|
|
1
1
|
/* Utilities for `fetch` when using range requests. It also allows you to handle errors easier */
|
2
2
|
|
3
|
-
import
|
3
|
+
import * as cache from './cache.js';
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
export interface CacheOptions {
|
8
|
-
/**
|
9
|
-
* If true, use multiple buffers to cache a file.
|
10
|
-
* This is useful when working with small parts of large files,
|
11
|
-
* since we don't need to allocate a large buffer that is mostly unused
|
12
|
-
* @default true
|
13
|
-
*/
|
14
|
-
sparse?: boolean;
|
15
|
-
|
16
|
-
/**
|
17
|
-
* The threshold for whether to combine regions or not
|
18
|
-
* @see CacheRegion
|
19
|
-
* @default 0xfff // 4 KiB
|
20
|
-
*/
|
21
|
-
regionGapThreshold?: number;
|
22
|
-
|
23
|
-
/**
|
24
|
-
* Whether to only update the cache when changing or deleting resources
|
25
|
-
* @default false
|
26
|
-
*/
|
27
|
-
cacheOnly?: boolean;
|
28
|
-
}
|
29
|
-
|
30
|
-
/**
|
31
|
-
* @deprecated Use `CacheOptions`
|
32
|
-
*/
|
33
|
-
export type ResourceCacheOptions = CacheOptions;
|
34
|
-
|
35
|
-
export type CacheRange = { start: number; end: number };
|
36
|
-
|
37
|
-
export interface CacheRegion {
|
38
|
-
/** The region's offset from the start of the resource */
|
39
|
-
offset: number;
|
40
|
-
|
41
|
-
/** Ranges cached in this region. These are absolute! */
|
42
|
-
ranges: CacheRange[];
|
43
|
-
|
44
|
-
/** Data for this region */
|
45
|
-
data: Uint8Array;
|
46
|
-
}
|
47
|
-
|
48
|
-
/**
|
49
|
-
* The cache for a specific resource
|
50
|
-
* @internal
|
51
|
-
*/
|
52
|
-
export class ResourceCache {
|
53
|
-
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
54
|
-
public readonly regions: CacheRegion[] = [];
|
55
|
-
|
56
|
-
public constructor(
|
57
|
-
/** The resource URL */
|
58
|
-
public readonly url: string,
|
59
|
-
/** The full size of the resource */
|
60
|
-
public readonly size: number,
|
61
|
-
protected readonly options: CacheOptions
|
62
|
-
) {
|
63
|
-
options.sparse ??= true;
|
64
|
-
if (!options.sparse) this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
65
|
-
|
66
|
-
resourcesCache.set(url, this);
|
67
|
-
}
|
68
|
-
|
69
|
-
/** Combines adjacent regions and combines adjacent ranges within a region */
|
70
|
-
public collect(): void {
|
71
|
-
if (!this.options.sparse) return;
|
72
|
-
const { regionGapThreshold = 0xfff } = this.options;
|
73
|
-
|
74
|
-
for (let i = 0; i < this.regions.length - 1; ) {
|
75
|
-
const current = this.regions[i];
|
76
|
-
const next = this.regions[i + 1];
|
77
|
-
|
78
|
-
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
79
|
-
i++;
|
80
|
-
continue;
|
81
|
-
}
|
82
|
-
|
83
|
-
// Combine ranges
|
84
|
-
current.ranges.push(...next.ranges);
|
85
|
-
current.ranges.sort((a, b) => a.start - b.start);
|
86
|
-
|
87
|
-
// Combine overlapping/adjacent ranges
|
88
|
-
current.ranges = current.ranges.reduce((acc: CacheRange[], range) => {
|
89
|
-
if (!acc.length || acc.at(-1)!.end < range.start) {
|
90
|
-
acc.push(range);
|
91
|
-
} else {
|
92
|
-
acc.at(-1)!.end = Math.max(acc.at(-1)!.end, range.end);
|
93
|
-
}
|
94
|
-
return acc;
|
95
|
-
}, []);
|
96
|
-
|
97
|
-
// Extend buffer to include the new region
|
98
|
-
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
99
|
-
current.data.set(next.data, next.offset - current.offset);
|
100
|
-
|
101
|
-
// Remove the next region after merging
|
102
|
-
this.regions.splice(i + 1, 1);
|
103
|
-
}
|
104
|
-
}
|
5
|
+
// Compatibility
|
105
6
|
|
106
|
-
|
107
|
-
|
108
|
-
|
7
|
+
/** @deprecated Use `cache.Options` */
|
8
|
+
export type ResourceCacheOptions = cache.Options;
|
9
|
+
/** @deprecated Use `cache.Resource` */
|
10
|
+
export const ResourceCache = cache.Resource;
|
11
|
+
/** @deprecated Use `cache.Resource` */
|
12
|
+
export type ResourceCache = cache.Resource;
|
13
|
+
/** @deprecated Use `cache.Range` */
|
14
|
+
export type CacheRange = cache.Range;
|
15
|
+
/** @deprecated Use `cache.Options` */
|
16
|
+
export type CacheOptions = cache.Options;
|
109
17
|
|
110
|
-
|
111
|
-
if (region.offset >= end) break;
|
112
|
-
|
113
|
-
for (const range of region.ranges) {
|
114
|
-
if (range.end <= start) continue;
|
115
|
-
|
116
|
-
if (range.start >= end) break;
|
117
|
-
|
118
|
-
if (range.start > start) {
|
119
|
-
missingRanges.push({ start, end: Math.min(range.start, end) });
|
120
|
-
}
|
121
|
-
|
122
|
-
// Adjust the current start if the region overlaps
|
123
|
-
if (range.end > start) start = Math.max(start, range.end);
|
124
|
-
|
125
|
-
if (start >= end) break;
|
126
|
-
}
|
127
|
-
|
128
|
-
if (start >= end) break;
|
129
|
-
}
|
130
|
-
|
131
|
-
// If there are still missing parts at the end
|
132
|
-
if (start < end) missingRanges.push({ start, end });
|
133
|
-
|
134
|
-
return missingRanges;
|
135
|
-
}
|
136
|
-
|
137
|
-
/** Get the region who's ranges include an offset */
|
138
|
-
public regionAt(offset: number): CacheRegion | undefined {
|
139
|
-
if (!this.regions.length) return;
|
140
|
-
|
141
|
-
for (const region of this.regions) {
|
142
|
-
if (region.offset > offset) break;
|
143
|
-
|
144
|
-
// Check if the offset is within this region
|
145
|
-
if (offset >= region.offset && offset < region.offset + region.data.byteLength) return region;
|
146
|
-
}
|
147
|
-
}
|
148
|
-
|
149
|
-
/** Add new data to the cache at given specified offset */
|
150
|
-
public add(data: Uint8Array, offset: number): this {
|
151
|
-
const end = offset + data.byteLength;
|
152
|
-
const region = this.regionAt(offset);
|
153
|
-
|
154
|
-
if (region) {
|
155
|
-
region.data = extendBuffer(region.data, end);
|
156
|
-
region.data.set(data, offset);
|
157
|
-
region.ranges.push({ start: offset, end });
|
158
|
-
region.ranges.sort((a, b) => a.start - b.start);
|
159
|
-
|
160
|
-
return this;
|
161
|
-
}
|
162
|
-
|
163
|
-
// Find the correct index to insert the new region
|
164
|
-
const newRegion: CacheRegion = { data, offset: offset, ranges: [{ start: offset, end }] };
|
165
|
-
const insertIndex = this.regions.findIndex(region => region.offset > offset);
|
166
|
-
|
167
|
-
// Insert at the right index to keep regions sorted
|
168
|
-
if (insertIndex == -1) {
|
169
|
-
this.regions.push(newRegion); // Append if no later region exists
|
170
|
-
} else {
|
171
|
-
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
172
|
-
}
|
173
|
-
|
174
|
-
return this;
|
175
|
-
}
|
176
|
-
}
|
18
|
+
/* eslint-disable @typescript-eslint/only-throw-error */
|
177
19
|
|
178
20
|
/**
|
179
21
|
* @internal
|
180
22
|
*/
|
181
|
-
export const resourcesCache = new Map<string,
|
23
|
+
export const resourcesCache = new Map<string, cache.Resource | undefined>();
|
182
24
|
|
183
25
|
export type Issue = { tag: 'status'; response: Response } | { tag: 'buffer'; response: Response; message: string } | { tag: 'fetch' | 'size'; message: string } | Error;
|
184
26
|
|
@@ -215,7 +57,7 @@ async function _fetch<const TBodyOptional extends boolean>(
|
|
215
57
|
return { response, data: raw ? new Uint8Array(raw) : undefined } as Fetched<TBodyOptional>;
|
216
58
|
}
|
217
59
|
|
218
|
-
export interface Options extends
|
60
|
+
export interface Options extends cache.Options {
|
219
61
|
/** Optionally provide a function for logging warnings */
|
220
62
|
warn?(message: string): unknown;
|
221
63
|
}
|
@@ -250,7 +92,7 @@ export async function get(url: string, options: GetOptions, init: RequestInit =
|
|
250
92
|
// Request no using ranges
|
251
93
|
if (typeof options.start != 'number' || typeof options.end != 'number') {
|
252
94
|
const { data } = await _fetch(url, init);
|
253
|
-
new
|
95
|
+
new cache.Resource(url, data.byteLength, options, resourcesCache).add(data, 0);
|
254
96
|
return data;
|
255
97
|
}
|
256
98
|
|
@@ -266,28 +108,28 @@ export async function get(url: string, options: GetOptions, init: RequestInit =
|
|
266
108
|
}
|
267
109
|
|
268
110
|
const { size, start, end } = options;
|
269
|
-
const
|
111
|
+
const resource = resourcesCache.get(url) ?? new cache.Resource(url, size, options, resourcesCache);
|
270
112
|
|
271
113
|
req.headers.set('If-Range', new Date().toUTCString());
|
272
114
|
|
273
|
-
for (const { start: from, end: to } of
|
115
|
+
for (const { start: from, end: to } of resource.missing(start, end)) {
|
274
116
|
const { data, response } = await _fetch(req, { headers: { Range: `bytes=${from}-${to}` } });
|
275
117
|
|
276
118
|
if (response.status == 206) {
|
277
|
-
|
119
|
+
resource.add(data, from);
|
278
120
|
continue;
|
279
121
|
}
|
280
122
|
|
281
123
|
// The first response doesn't have a "partial content" (206) status
|
282
124
|
options.warn?.(url + ': Remote does not support range requests with bytes. Falling back to full data.');
|
283
|
-
new
|
125
|
+
new cache.Resource(url, size, options, resourcesCache).add(data, 0);
|
284
126
|
return data.subarray(start, end);
|
285
127
|
}
|
286
128
|
|
287
129
|
// This ensures we get a single buffer with the entire requested range
|
288
|
-
|
130
|
+
resource.collect();
|
289
131
|
|
290
|
-
const region =
|
132
|
+
const region = resource.regionAt(start)!;
|
291
133
|
return region.data.subarray(start - region.offset, end - region.offset);
|
292
134
|
}
|
293
135
|
|
@@ -300,7 +142,7 @@ export const GET = get;
|
|
300
142
|
* Synchronously gets a cached resource
|
301
143
|
* Assumes you pass valid start and end when using ranges
|
302
144
|
*/
|
303
|
-
export function getCached(url: string, options: GetOptions): { data?: Uint8Array; missing:
|
145
|
+
export function getCached(url: string, options: GetOptions): { data?: Uint8Array; missing: cache.Range[] } {
|
304
146
|
const cache = resourcesCache.get(url);
|
305
147
|
|
306
148
|
/**
|
@@ -350,16 +192,16 @@ interface SetOptions extends Options {
|
|
350
192
|
*/
|
351
193
|
export async function set(url: string, data: Uint8Array, options: SetOptions, init: RequestInit = {}): Promise<void> {
|
352
194
|
if (!resourcesCache.has(url)) {
|
353
|
-
new
|
195
|
+
new cache.Resource(url, options.size ?? data.byteLength, options, resourcesCache);
|
354
196
|
}
|
355
197
|
|
356
|
-
const
|
198
|
+
const resource = resourcesCache.get(url)!;
|
357
199
|
|
358
200
|
const { offset = 0 } = options;
|
359
201
|
|
360
202
|
if (!options.cacheOnly) await _fetch(new Request(url, init), { method: 'POST' }, true);
|
361
203
|
|
362
|
-
|
204
|
+
resource.add(data, offset).collect();
|
363
205
|
}
|
364
206
|
|
365
207
|
/**
|