utilium 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cache.d.ts +58 -0
- package/dist/cache.js +121 -0
- package/dist/debugging.d.ts +17 -0
- package/dist/debugging.js +29 -6
- package/dist/requests.d.ts +54 -35
- package/dist/requests.js +57 -132
- package/package.json +1 -1
- package/readme.md +1 -0
- package/src/cache.ts +169 -0
- package/src/debugging.ts +45 -5
- package/src/requests.ts +105 -178
package/dist/cache.d.ts
ADDED
@@ -0,0 +1,58 @@
|
|
1
|
+
export interface Options {
|
2
|
+
/**
|
3
|
+
* If true, use multiple buffers to cache a file.
|
4
|
+
* This is useful when working with small parts of large files,
|
5
|
+
* since we don't need to allocate a large buffer that is mostly unused
|
6
|
+
* @default true
|
7
|
+
*/
|
8
|
+
sparse?: boolean;
|
9
|
+
/**
|
10
|
+
* The threshold for whether to combine regions or not
|
11
|
+
* @see Region
|
12
|
+
* @default 0xfff // 4 KiB
|
13
|
+
*/
|
14
|
+
regionGapThreshold?: number;
|
15
|
+
/**
|
16
|
+
* Whether to only update the cache when changing or deleting resources
|
17
|
+
* @default false
|
18
|
+
*/
|
19
|
+
cacheOnly?: boolean;
|
20
|
+
}
|
21
|
+
export type Range = {
|
22
|
+
start: number;
|
23
|
+
end: number;
|
24
|
+
};
|
25
|
+
export interface Region {
|
26
|
+
/** The region's offset from the start of the resource */
|
27
|
+
offset: number;
|
28
|
+
/** Ranges cached in this region. These are absolute! */
|
29
|
+
ranges: Range[];
|
30
|
+
/** Data for this region */
|
31
|
+
data: Uint8Array;
|
32
|
+
}
|
33
|
+
/**
|
34
|
+
* The cache for a specific resource
|
35
|
+
* @internal
|
36
|
+
*/
|
37
|
+
export declare class Resource {
|
38
|
+
/** The resource ID */
|
39
|
+
readonly id: string;
|
40
|
+
/** The full size of the resource */
|
41
|
+
readonly size: number;
|
42
|
+
protected readonly options: Options;
|
43
|
+
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
44
|
+
readonly regions: Region[];
|
45
|
+
constructor(
|
46
|
+
/** The resource ID */
|
47
|
+
id: string,
|
48
|
+
/** The full size of the resource */
|
49
|
+
size: number, options: Options, resources?: Map<string, Resource | undefined>);
|
50
|
+
/** Combines adjacent regions and combines adjacent ranges within a region */
|
51
|
+
collect(): void;
|
52
|
+
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
53
|
+
missing(start: number, end: number): Range[];
|
54
|
+
/** Get the region who's ranges include an offset */
|
55
|
+
regionAt(offset: number): Region | undefined;
|
56
|
+
/** Add new data to the cache at given specified offset */
|
57
|
+
add(data: Uint8Array, offset: number): this;
|
58
|
+
}
|
package/dist/cache.js
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
/** A ranged cache */
|
2
|
+
import { extendBuffer } from './buffer.js';
|
3
|
+
/**
|
4
|
+
* The cache for a specific resource
|
5
|
+
* @internal
|
6
|
+
*/
|
7
|
+
export class Resource {
|
8
|
+
id;
|
9
|
+
size;
|
10
|
+
options;
|
11
|
+
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
12
|
+
regions = [];
|
13
|
+
constructor(
|
14
|
+
/** The resource ID */
|
15
|
+
id,
|
16
|
+
/** The full size of the resource */
|
17
|
+
size, options, resources) {
|
18
|
+
this.id = id;
|
19
|
+
this.size = size;
|
20
|
+
this.options = options;
|
21
|
+
options.sparse ??= true;
|
22
|
+
if (!options.sparse)
|
23
|
+
this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
24
|
+
resources?.set(id, this);
|
25
|
+
}
|
26
|
+
/** Combines adjacent regions and combines adjacent ranges within a region */
|
27
|
+
collect() {
|
28
|
+
if (!this.options.sparse)
|
29
|
+
return;
|
30
|
+
const { regionGapThreshold = 0xfff } = this.options;
|
31
|
+
for (let i = 0; i < this.regions.length - 1;) {
|
32
|
+
const current = this.regions[i];
|
33
|
+
const next = this.regions[i + 1];
|
34
|
+
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
35
|
+
i++;
|
36
|
+
continue;
|
37
|
+
}
|
38
|
+
// Combine ranges
|
39
|
+
current.ranges.push(...next.ranges);
|
40
|
+
current.ranges.sort((a, b) => a.start - b.start);
|
41
|
+
// Combine overlapping/adjacent ranges
|
42
|
+
current.ranges = current.ranges.reduce((acc, range) => {
|
43
|
+
if (!acc.length || acc.at(-1).end < range.start) {
|
44
|
+
acc.push(range);
|
45
|
+
}
|
46
|
+
else {
|
47
|
+
acc.at(-1).end = Math.max(acc.at(-1).end, range.end);
|
48
|
+
}
|
49
|
+
return acc;
|
50
|
+
}, []);
|
51
|
+
// Extend buffer to include the new region
|
52
|
+
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
53
|
+
current.data.set(next.data, next.offset - current.offset);
|
54
|
+
// Remove the next region after merging
|
55
|
+
this.regions.splice(i + 1, 1);
|
56
|
+
}
|
57
|
+
}
|
58
|
+
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
59
|
+
missing(start, end) {
|
60
|
+
const missingRanges = [];
|
61
|
+
for (const region of this.regions) {
|
62
|
+
if (region.offset >= end)
|
63
|
+
break;
|
64
|
+
for (const range of region.ranges) {
|
65
|
+
if (range.end <= start)
|
66
|
+
continue;
|
67
|
+
if (range.start >= end)
|
68
|
+
break;
|
69
|
+
if (range.start > start) {
|
70
|
+
missingRanges.push({ start, end: Math.min(range.start, end) });
|
71
|
+
}
|
72
|
+
// Adjust the current start if the region overlaps
|
73
|
+
if (range.end > start)
|
74
|
+
start = Math.max(start, range.end);
|
75
|
+
if (start >= end)
|
76
|
+
break;
|
77
|
+
}
|
78
|
+
if (start >= end)
|
79
|
+
break;
|
80
|
+
}
|
81
|
+
// If there are still missing parts at the end
|
82
|
+
if (start < end)
|
83
|
+
missingRanges.push({ start, end });
|
84
|
+
return missingRanges;
|
85
|
+
}
|
86
|
+
/** Get the region who's ranges include an offset */
|
87
|
+
regionAt(offset) {
|
88
|
+
if (!this.regions.length)
|
89
|
+
return;
|
90
|
+
for (const region of this.regions) {
|
91
|
+
if (region.offset > offset)
|
92
|
+
break;
|
93
|
+
// Check if the offset is within this region
|
94
|
+
if (offset >= region.offset && offset < region.offset + region.data.byteLength)
|
95
|
+
return region;
|
96
|
+
}
|
97
|
+
}
|
98
|
+
/** Add new data to the cache at given specified offset */
|
99
|
+
add(data, offset) {
|
100
|
+
const end = offset + data.byteLength;
|
101
|
+
const region = this.regionAt(offset);
|
102
|
+
if (region) {
|
103
|
+
region.data = extendBuffer(region.data, end);
|
104
|
+
region.data.set(data, offset);
|
105
|
+
region.ranges.push({ start: offset, end });
|
106
|
+
region.ranges.sort((a, b) => a.start - b.start);
|
107
|
+
return this;
|
108
|
+
}
|
109
|
+
// Find the correct index to insert the new region
|
110
|
+
const newRegion = { data, offset: offset, ranges: [{ start: offset, end }] };
|
111
|
+
const insertIndex = this.regions.findIndex(region => region.offset > offset);
|
112
|
+
// Insert at the right index to keep regions sorted
|
113
|
+
if (insertIndex == -1) {
|
114
|
+
this.regions.push(newRegion); // Append if no later region exists
|
115
|
+
}
|
116
|
+
else {
|
117
|
+
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
118
|
+
}
|
119
|
+
return this;
|
120
|
+
}
|
121
|
+
}
|
package/dist/debugging.d.ts
CHANGED
@@ -1,8 +1,25 @@
|
|
1
1
|
export interface CreateLoggerOptions {
|
2
|
+
/**
|
3
|
+
* The function used to output
|
4
|
+
* @default console.log
|
5
|
+
*/
|
2
6
|
output?: (...args: any[]) => void;
|
3
7
|
stringify?: (value: unknown) => string;
|
8
|
+
/**
|
9
|
+
* Whether to output the class name when logging methods
|
10
|
+
* @default true
|
11
|
+
*/
|
4
12
|
className?: boolean;
|
13
|
+
/**
|
14
|
+
* The separator used to separate the class name and method.
|
15
|
+
* Ignored if `className` is `false`
|
16
|
+
* @default '#'
|
17
|
+
*/
|
5
18
|
separator?: string;
|
19
|
+
/**
|
20
|
+
* Whether to log the return value
|
21
|
+
* @default false
|
22
|
+
*/
|
6
23
|
returnValue?: boolean;
|
7
24
|
}
|
8
25
|
type LoggableDecoratorContext = Exclude<DecoratorContext, ClassFieldDecoratorContext>;
|
package/dist/debugging.js
CHANGED
@@ -1,10 +1,33 @@
|
|
1
1
|
function defaultStringify(value) {
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
2
|
+
switch (typeof value) {
|
3
|
+
case 'undefined':
|
4
|
+
return 'undefined';
|
5
|
+
case 'string':
|
6
|
+
case 'number':
|
7
|
+
case 'boolean':
|
8
|
+
return JSON.stringify(value);
|
9
|
+
case 'bigint':
|
10
|
+
case 'symbol':
|
11
|
+
return value.toString();
|
12
|
+
case 'function':
|
13
|
+
return value.name + '()';
|
14
|
+
case 'object':
|
15
|
+
if (value === null)
|
16
|
+
return 'null';
|
17
|
+
if (ArrayBuffer.isView(value)) {
|
18
|
+
const length = 'length' in value ? value.length : value.byteLength / value.constructor.BYTES_PER_ELEMENT;
|
19
|
+
return `${value.constructor.name.replaceAll('Array', '').toLowerCase()}[${length}]`;
|
20
|
+
}
|
21
|
+
if (Array.isArray(value))
|
22
|
+
return `unknown[${value.length}]`;
|
23
|
+
try {
|
24
|
+
const json = JSON.stringify(value);
|
25
|
+
return json.length < 100 ? json : value.toString();
|
26
|
+
}
|
27
|
+
catch {
|
28
|
+
return value.toString();
|
29
|
+
}
|
30
|
+
}
|
8
31
|
}
|
9
32
|
/**
|
10
33
|
* Create a function that can be used to decorate classes and non-field members.
|
package/dist/requests.d.ts
CHANGED
@@ -1,31 +1,19 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
export type
|
17
|
-
start: number;
|
18
|
-
end: number;
|
19
|
-
};
|
20
|
-
export interface CacheRegion {
|
21
|
-
/** The region's offset from the start of the resource */
|
22
|
-
offset: number;
|
23
|
-
/** Ranges cached in this region. These are absolute! */
|
24
|
-
ranges: CacheRange[];
|
25
|
-
/** Data for this region */
|
26
|
-
data: Uint8Array;
|
27
|
-
}
|
28
|
-
export type RequestError = {
|
1
|
+
import * as cache from './cache.js';
|
2
|
+
/** @deprecated Use `cache.Options` */
|
3
|
+
export type ResourceCacheOptions = cache.Options;
|
4
|
+
/** @deprecated Use `cache.Resource` */
|
5
|
+
export declare const ResourceCache: typeof cache.Resource;
|
6
|
+
/** @deprecated Use `cache.Resource` */
|
7
|
+
export type ResourceCache = cache.Resource;
|
8
|
+
/** @deprecated Use `cache.Range` */
|
9
|
+
export type CacheRange = cache.Range;
|
10
|
+
/** @deprecated Use `cache.Options` */
|
11
|
+
export type CacheOptions = cache.Options;
|
12
|
+
/**
|
13
|
+
* @internal
|
14
|
+
*/
|
15
|
+
export declare const resourcesCache: Map<string, cache.Resource | undefined>;
|
16
|
+
export type Issue = {
|
29
17
|
tag: 'status';
|
30
18
|
response: Response;
|
31
19
|
} | {
|
@@ -36,7 +24,19 @@ export type RequestError = {
|
|
36
24
|
tag: 'fetch' | 'size';
|
37
25
|
message: string;
|
38
26
|
} | Error;
|
39
|
-
|
27
|
+
/**
|
28
|
+
* @deprecated Use `Issue`
|
29
|
+
*/
|
30
|
+
export type RequestError = Issue;
|
31
|
+
export interface Options extends cache.Options {
|
32
|
+
/** Optionally provide a function for logging warnings */
|
33
|
+
warn?(message: string): unknown;
|
34
|
+
}
|
35
|
+
/**
|
36
|
+
* @deprecated Use `Options`
|
37
|
+
*/
|
38
|
+
export type RequestOptions = Options;
|
39
|
+
export interface GetOptions extends Options {
|
40
40
|
/**
|
41
41
|
* When using range requests,
|
42
42
|
* a HEAD request must normally be used to determine the full size of the resource.
|
@@ -47,19 +47,38 @@ export interface RequestOptions extends ResourceCacheOptions {
|
|
47
47
|
start?: number;
|
48
48
|
/** The end of the range */
|
49
49
|
end?: number;
|
50
|
-
/** Optionally provide a function for logging warnings */
|
51
|
-
warn?(message: string): unknown;
|
52
50
|
}
|
53
51
|
/**
|
54
52
|
* Make a GET request without worrying about ranges
|
55
53
|
* @throws RequestError
|
56
54
|
*/
|
57
|
-
export declare function
|
55
|
+
export declare function get(url: string, options: GetOptions, init?: RequestInit): Promise<Uint8Array>;
|
56
|
+
/**
|
57
|
+
* @deprecated Use `get`
|
58
|
+
*/
|
59
|
+
export declare const GET: typeof get;
|
58
60
|
/**
|
59
61
|
* Synchronously gets a cached resource
|
60
62
|
* Assumes you pass valid start and end when using ranges
|
61
63
|
*/
|
62
|
-
export declare function getCached(url: string, options:
|
63
|
-
data
|
64
|
-
missing:
|
64
|
+
export declare function getCached(url: string, options: GetOptions): {
|
65
|
+
data?: Uint8Array;
|
66
|
+
missing: cache.Range[];
|
65
67
|
};
|
68
|
+
interface SetOptions extends Options {
|
69
|
+
/** The offset we are updating at */
|
70
|
+
offset?: number;
|
71
|
+
/** If a cache for the resource doesn't exist, this will be used as the full size */
|
72
|
+
size?: number;
|
73
|
+
}
|
74
|
+
/**
|
75
|
+
* Make a POST request to set (or create) data on the server and update the cache.
|
76
|
+
* @throws RequestError
|
77
|
+
*/
|
78
|
+
export declare function set(url: string, data: Uint8Array, options: SetOptions, init?: RequestInit): Promise<void>;
|
79
|
+
/**
|
80
|
+
* Make a DELETE request to remove the resource from the server and clear it from the cache.
|
81
|
+
* @throws RequestError
|
82
|
+
*/
|
83
|
+
export declare function remove(url: string, options?: Options, init?: RequestInit): Promise<void>;
|
84
|
+
export {};
|
package/dist/requests.js
CHANGED
@@ -1,145 +1,39 @@
|
|
1
1
|
/* Utilities for `fetch` when using range requests. It also allows you to handle errors easier */
|
2
|
-
import
|
3
|
-
/**
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
constructor(
|
11
|
-
/** The resource URL */
|
12
|
-
url,
|
13
|
-
/** The full size of the resource */
|
14
|
-
size, options) {
|
15
|
-
this.url = url;
|
16
|
-
this.size = size;
|
17
|
-
this.options = options;
|
18
|
-
options.sparse ??= true;
|
19
|
-
if (!options.sparse)
|
20
|
-
this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
21
|
-
requestsCache.set(url, this);
|
22
|
-
}
|
23
|
-
/** Combines adjacent regions and combines adjacent ranges within a region */
|
24
|
-
collect() {
|
25
|
-
if (!this.options.sparse)
|
26
|
-
return;
|
27
|
-
const { regionGapThreshold = 0xfff } = this.options;
|
28
|
-
for (let i = 0; i < this.regions.length - 1;) {
|
29
|
-
const current = this.regions[i];
|
30
|
-
const next = this.regions[i + 1];
|
31
|
-
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
32
|
-
i++;
|
33
|
-
continue;
|
34
|
-
}
|
35
|
-
// Combine ranges
|
36
|
-
current.ranges.push(...next.ranges);
|
37
|
-
current.ranges.sort((a, b) => a.start - b.start);
|
38
|
-
// Combine overlapping/adjacent ranges
|
39
|
-
current.ranges = current.ranges.reduce((acc, range) => {
|
40
|
-
if (!acc.length || acc.at(-1).end < range.start) {
|
41
|
-
acc.push(range);
|
42
|
-
}
|
43
|
-
else {
|
44
|
-
acc.at(-1).end = Math.max(acc.at(-1).end, range.end);
|
45
|
-
}
|
46
|
-
return acc;
|
47
|
-
}, []);
|
48
|
-
// Extend buffer to include the new region
|
49
|
-
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
50
|
-
current.data.set(next.data, next.offset - current.offset);
|
51
|
-
// Remove the next region after merging
|
52
|
-
this.regions.splice(i + 1, 1);
|
53
|
-
}
|
54
|
-
}
|
55
|
-
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
56
|
-
missing(start, end) {
|
57
|
-
const missingRanges = [];
|
58
|
-
for (const region of this.regions) {
|
59
|
-
if (region.offset >= end)
|
60
|
-
break;
|
61
|
-
for (const range of region.ranges) {
|
62
|
-
if (range.end <= start)
|
63
|
-
continue;
|
64
|
-
if (range.start >= end)
|
65
|
-
break;
|
66
|
-
if (range.start > start) {
|
67
|
-
missingRanges.push({ start, end: Math.min(range.start, end) });
|
68
|
-
}
|
69
|
-
// Adjust the current start if the region overlaps
|
70
|
-
if (range.end > start)
|
71
|
-
start = Math.max(start, range.end);
|
72
|
-
if (start >= end)
|
73
|
-
break;
|
74
|
-
}
|
75
|
-
if (start >= end)
|
76
|
-
break;
|
77
|
-
}
|
78
|
-
// If there are still missing parts at the end
|
79
|
-
if (start < end)
|
80
|
-
missingRanges.push({ start, end });
|
81
|
-
return missingRanges;
|
82
|
-
}
|
83
|
-
/** Get the region who's ranges include an offset */
|
84
|
-
regionAt(offset) {
|
85
|
-
if (!this.regions.length)
|
86
|
-
return;
|
87
|
-
for (const region of this.regions) {
|
88
|
-
if (region.offset > offset)
|
89
|
-
break;
|
90
|
-
// Check if the offset is within this region
|
91
|
-
if (offset >= region.offset && offset < region.offset + region.data.byteLength)
|
92
|
-
return region;
|
93
|
-
}
|
94
|
-
}
|
95
|
-
/** Add new data to the cache at given specified offset */
|
96
|
-
add(data, start, end = start + data.byteLength) {
|
97
|
-
const region = this.regionAt(start);
|
98
|
-
if (region) {
|
99
|
-
region.data = extendBuffer(region.data, end);
|
100
|
-
region.ranges.push({ start, end });
|
101
|
-
region.ranges.sort((a, b) => a.start - b.start);
|
102
|
-
return this;
|
103
|
-
}
|
104
|
-
// Find the correct index to insert the new region
|
105
|
-
const newRegion = { data, offset: start, ranges: [{ start, end }] };
|
106
|
-
const insertIndex = this.regions.findIndex(region => region.offset > start);
|
107
|
-
// Insert at the right index to keep regions sorted
|
108
|
-
if (insertIndex == -1) {
|
109
|
-
this.regions.push(newRegion); // Append if no later region exists
|
110
|
-
}
|
111
|
-
else {
|
112
|
-
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
113
|
-
}
|
114
|
-
return this;
|
115
|
-
}
|
116
|
-
}
|
117
|
-
const requestsCache = new Map();
|
2
|
+
import * as cache from './cache.js';
|
3
|
+
/** @deprecated Use `cache.Resource` */
|
4
|
+
export const ResourceCache = cache.Resource;
|
5
|
+
/* eslint-disable @typescript-eslint/only-throw-error */
|
6
|
+
/**
|
7
|
+
* @internal
|
8
|
+
*/
|
9
|
+
export const resourcesCache = new Map();
|
118
10
|
/**
|
119
11
|
* Wraps `fetch`
|
120
12
|
* @throws RequestError
|
121
13
|
*/
|
122
|
-
async function
|
14
|
+
async function _fetch(input, init = {}, bodyOptional = false) {
|
123
15
|
const response = await fetch(input, init).catch((error) => {
|
124
16
|
throw { tag: 'fetch', message: error.message };
|
125
17
|
});
|
126
18
|
if (!response.ok)
|
127
19
|
throw { tag: 'status', response };
|
128
|
-
const
|
20
|
+
const raw = await response.arrayBuffer().catch((error) => {
|
21
|
+
if (bodyOptional)
|
22
|
+
return;
|
129
23
|
throw { tag: 'buffer', response, message: error.message };
|
130
24
|
});
|
131
|
-
return { response, data: new Uint8Array(
|
25
|
+
return { response, data: raw ? new Uint8Array(raw) : undefined };
|
132
26
|
}
|
133
27
|
/**
|
134
28
|
* Make a GET request without worrying about ranges
|
135
29
|
* @throws RequestError
|
136
30
|
*/
|
137
|
-
export async function
|
31
|
+
export async function get(url, options, init = {}) {
|
138
32
|
const req = new Request(url, init);
|
139
33
|
// Request no using ranges
|
140
34
|
if (typeof options.start != 'number' || typeof options.end != 'number') {
|
141
|
-
const { data } = await
|
142
|
-
new
|
35
|
+
const { data } = await _fetch(url, init);
|
36
|
+
new cache.Resource(url, data.byteLength, options, resourcesCache).add(data, 0);
|
143
37
|
return data;
|
144
38
|
}
|
145
39
|
// Range requests
|
@@ -152,35 +46,43 @@ export async function GET(url, options, init = {}) {
|
|
152
46
|
options.size = size;
|
153
47
|
}
|
154
48
|
const { size, start, end } = options;
|
155
|
-
const
|
49
|
+
const resource = resourcesCache.get(url) ?? new cache.Resource(url, size, options, resourcesCache);
|
156
50
|
req.headers.set('If-Range', new Date().toUTCString());
|
157
|
-
for (const { start: from, end: to } of
|
158
|
-
const { data, response } = await
|
51
|
+
for (const { start: from, end: to } of resource.missing(start, end)) {
|
52
|
+
const { data, response } = await _fetch(req, { headers: { Range: `bytes=${from}-${to}` } });
|
159
53
|
if (response.status == 206) {
|
160
|
-
|
54
|
+
resource.add(data, from);
|
161
55
|
continue;
|
162
56
|
}
|
163
57
|
// The first response doesn't have a "partial content" (206) status
|
164
58
|
options.warn?.(url + ': Remote does not support range requests with bytes. Falling back to full data.');
|
165
|
-
new
|
59
|
+
new cache.Resource(url, size, options, resourcesCache).add(data, 0);
|
166
60
|
return data.subarray(start, end);
|
167
61
|
}
|
168
62
|
// This ensures we get a single buffer with the entire requested range
|
169
|
-
|
170
|
-
const region =
|
63
|
+
resource.collect();
|
64
|
+
const region = resource.regionAt(start);
|
171
65
|
return region.data.subarray(start - region.offset, end - region.offset);
|
172
66
|
}
|
67
|
+
/**
|
68
|
+
* @deprecated Use `get`
|
69
|
+
*/
|
70
|
+
export const GET = get;
|
173
71
|
/**
|
174
72
|
* Synchronously gets a cached resource
|
175
73
|
* Assumes you pass valid start and end when using ranges
|
176
74
|
*/
|
177
75
|
export function getCached(url, options) {
|
178
|
-
const cache =
|
76
|
+
const cache = resourcesCache.get(url);
|
179
77
|
/**
|
180
78
|
* @todo Make sure we have a size?
|
181
79
|
*/
|
182
|
-
if (!cache)
|
183
|
-
|
80
|
+
if (!cache) {
|
81
|
+
if (options.size)
|
82
|
+
return { data: new Uint8Array(0), missing: [{ start: 0, end: options.size ?? 0 }] };
|
83
|
+
options.warn?.(url + ': Size not provided and cache is empty, can not determine missing range');
|
84
|
+
return { data: undefined, missing: [] };
|
85
|
+
}
|
184
86
|
const { start = 0, end = cache.size } = options;
|
185
87
|
const data = new Uint8Array(end - start);
|
186
88
|
for (const region of cache.regions) {
|
@@ -202,3 +104,26 @@ export function getCached(url, options) {
|
|
202
104
|
}
|
203
105
|
return { data, missing: cache.missing(start, end) };
|
204
106
|
}
|
107
|
+
/**
|
108
|
+
* Make a POST request to set (or create) data on the server and update the cache.
|
109
|
+
* @throws RequestError
|
110
|
+
*/
|
111
|
+
export async function set(url, data, options, init = {}) {
|
112
|
+
if (!resourcesCache.has(url)) {
|
113
|
+
new cache.Resource(url, options.size ?? data.byteLength, options, resourcesCache);
|
114
|
+
}
|
115
|
+
const resource = resourcesCache.get(url);
|
116
|
+
const { offset = 0 } = options;
|
117
|
+
if (!options.cacheOnly)
|
118
|
+
await _fetch(new Request(url, init), { method: 'POST' }, true);
|
119
|
+
resource.add(data, offset).collect();
|
120
|
+
}
|
121
|
+
/**
|
122
|
+
* Make a DELETE request to remove the resource from the server and clear it from the cache.
|
123
|
+
* @throws RequestError
|
124
|
+
*/
|
125
|
+
export async function remove(url, options = {}, init = {}) {
|
126
|
+
if (!options.cacheOnly)
|
127
|
+
await _fetch(new Request(url, init), { method: 'DELETE' }, true);
|
128
|
+
resourcesCache.delete(url);
|
129
|
+
}
|
package/package.json
CHANGED
package/readme.md
CHANGED
@@ -9,6 +9,7 @@ A bunch of utilities for Typescript. This includes:
|
|
9
9
|
- Extending buffers easier with `extendBuffer`
|
10
10
|
- Convenience types and functions for strings and objects
|
11
11
|
- RNG functions
|
12
|
+
- Ranged cache
|
12
13
|
- `List`, a class that combines the best aspects of `Set` and arrays
|
13
14
|
- `JSONFileMap` and `FolderMap`
|
14
15
|
- Version utilities
|
package/src/cache.ts
ADDED
@@ -0,0 +1,169 @@
|
|
1
|
+
/** A ranged cache */
|
2
|
+
import { extendBuffer } from './buffer.js';
|
3
|
+
|
4
|
+
export interface Options {
|
5
|
+
/**
|
6
|
+
* If true, use multiple buffers to cache a file.
|
7
|
+
* This is useful when working with small parts of large files,
|
8
|
+
* since we don't need to allocate a large buffer that is mostly unused
|
9
|
+
* @default true
|
10
|
+
*/
|
11
|
+
sparse?: boolean;
|
12
|
+
|
13
|
+
/**
|
14
|
+
* The threshold for whether to combine regions or not
|
15
|
+
* @see Region
|
16
|
+
* @default 0xfff // 4 KiB
|
17
|
+
*/
|
18
|
+
regionGapThreshold?: number;
|
19
|
+
|
20
|
+
/**
|
21
|
+
* Whether to only update the cache when changing or deleting resources
|
22
|
+
* @default false
|
23
|
+
*/
|
24
|
+
cacheOnly?: boolean;
|
25
|
+
}
|
26
|
+
|
27
|
+
export type Range = { start: number; end: number };
|
28
|
+
|
29
|
+
export interface Region {
|
30
|
+
/** The region's offset from the start of the resource */
|
31
|
+
offset: number;
|
32
|
+
|
33
|
+
/** Ranges cached in this region. These are absolute! */
|
34
|
+
ranges: Range[];
|
35
|
+
|
36
|
+
/** Data for this region */
|
37
|
+
data: Uint8Array;
|
38
|
+
}
|
39
|
+
|
40
|
+
/**
|
41
|
+
* The cache for a specific resource
|
42
|
+
* @internal
|
43
|
+
*/
|
44
|
+
export class Resource {
|
45
|
+
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
46
|
+
public readonly regions: Region[] = [];
|
47
|
+
|
48
|
+
public constructor(
|
49
|
+
/** The resource ID */
|
50
|
+
public readonly id: string,
|
51
|
+
/** The full size of the resource */
|
52
|
+
public readonly size: number,
|
53
|
+
protected readonly options: Options,
|
54
|
+
resources?: Map<string, Resource | undefined>
|
55
|
+
) {
|
56
|
+
options.sparse ??= true;
|
57
|
+
if (!options.sparse) this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
58
|
+
|
59
|
+
resources?.set(id, this);
|
60
|
+
}
|
61
|
+
|
62
|
+
/** Combines adjacent regions and combines adjacent ranges within a region */
|
63
|
+
public collect(): void {
|
64
|
+
if (!this.options.sparse) return;
|
65
|
+
const { regionGapThreshold = 0xfff } = this.options;
|
66
|
+
|
67
|
+
for (let i = 0; i < this.regions.length - 1; ) {
|
68
|
+
const current = this.regions[i];
|
69
|
+
const next = this.regions[i + 1];
|
70
|
+
|
71
|
+
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
72
|
+
i++;
|
73
|
+
continue;
|
74
|
+
}
|
75
|
+
|
76
|
+
// Combine ranges
|
77
|
+
current.ranges.push(...next.ranges);
|
78
|
+
current.ranges.sort((a, b) => a.start - b.start);
|
79
|
+
|
80
|
+
// Combine overlapping/adjacent ranges
|
81
|
+
current.ranges = current.ranges.reduce((acc: Range[], range) => {
|
82
|
+
if (!acc.length || acc.at(-1)!.end < range.start) {
|
83
|
+
acc.push(range);
|
84
|
+
} else {
|
85
|
+
acc.at(-1)!.end = Math.max(acc.at(-1)!.end, range.end);
|
86
|
+
}
|
87
|
+
return acc;
|
88
|
+
}, []);
|
89
|
+
|
90
|
+
// Extend buffer to include the new region
|
91
|
+
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
92
|
+
current.data.set(next.data, next.offset - current.offset);
|
93
|
+
|
94
|
+
// Remove the next region after merging
|
95
|
+
this.regions.splice(i + 1, 1);
|
96
|
+
}
|
97
|
+
}
|
98
|
+
|
99
|
+
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
100
|
+
public missing(start: number, end: number): Range[] {
|
101
|
+
const missingRanges: Range[] = [];
|
102
|
+
|
103
|
+
for (const region of this.regions) {
|
104
|
+
if (region.offset >= end) break;
|
105
|
+
|
106
|
+
for (const range of region.ranges) {
|
107
|
+
if (range.end <= start) continue;
|
108
|
+
|
109
|
+
if (range.start >= end) break;
|
110
|
+
|
111
|
+
if (range.start > start) {
|
112
|
+
missingRanges.push({ start, end: Math.min(range.start, end) });
|
113
|
+
}
|
114
|
+
|
115
|
+
// Adjust the current start if the region overlaps
|
116
|
+
if (range.end > start) start = Math.max(start, range.end);
|
117
|
+
|
118
|
+
if (start >= end) break;
|
119
|
+
}
|
120
|
+
|
121
|
+
if (start >= end) break;
|
122
|
+
}
|
123
|
+
|
124
|
+
// If there are still missing parts at the end
|
125
|
+
if (start < end) missingRanges.push({ start, end });
|
126
|
+
|
127
|
+
return missingRanges;
|
128
|
+
}
|
129
|
+
|
130
|
+
/** Get the region who's ranges include an offset */
|
131
|
+
public regionAt(offset: number): Region | undefined {
|
132
|
+
if (!this.regions.length) return;
|
133
|
+
|
134
|
+
for (const region of this.regions) {
|
135
|
+
if (region.offset > offset) break;
|
136
|
+
|
137
|
+
// Check if the offset is within this region
|
138
|
+
if (offset >= region.offset && offset < region.offset + region.data.byteLength) return region;
|
139
|
+
}
|
140
|
+
}
|
141
|
+
|
142
|
+
/** Add new data to the cache at given specified offset */
|
143
|
+
public add(data: Uint8Array, offset: number): this {
|
144
|
+
const end = offset + data.byteLength;
|
145
|
+
const region = this.regionAt(offset);
|
146
|
+
|
147
|
+
if (region) {
|
148
|
+
region.data = extendBuffer(region.data, end);
|
149
|
+
region.data.set(data, offset);
|
150
|
+
region.ranges.push({ start: offset, end });
|
151
|
+
region.ranges.sort((a, b) => a.start - b.start);
|
152
|
+
|
153
|
+
return this;
|
154
|
+
}
|
155
|
+
|
156
|
+
// Find the correct index to insert the new region
|
157
|
+
const newRegion: Region = { data, offset: offset, ranges: [{ start: offset, end }] };
|
158
|
+
const insertIndex = this.regions.findIndex(region => region.offset > offset);
|
159
|
+
|
160
|
+
// Insert at the right index to keep regions sorted
|
161
|
+
if (insertIndex == -1) {
|
162
|
+
this.regions.push(newRegion); // Append if no later region exists
|
163
|
+
} else {
|
164
|
+
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
165
|
+
}
|
166
|
+
|
167
|
+
return this;
|
168
|
+
}
|
169
|
+
}
|
package/src/debugging.ts
CHANGED
@@ -1,17 +1,57 @@
|
|
1
|
-
/* eslint-disable @typescript-eslint/no-explicit-any */
|
1
|
+
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/no-base-to-string */
|
2
2
|
export interface CreateLoggerOptions {
|
3
|
+
/**
|
4
|
+
* The function used to output
|
5
|
+
* @default console.log
|
6
|
+
*/
|
3
7
|
output?: (...args: any[]) => void;
|
4
8
|
stringify?: (value: unknown) => string;
|
9
|
+
/**
|
10
|
+
* Whether to output the class name when logging methods
|
11
|
+
* @default true
|
12
|
+
*/
|
5
13
|
className?: boolean;
|
14
|
+
/**
|
15
|
+
* The separator used to separate the class name and method.
|
16
|
+
* Ignored if `className` is `false`
|
17
|
+
* @default '#'
|
18
|
+
*/
|
6
19
|
separator?: string;
|
20
|
+
/**
|
21
|
+
* Whether to log the return value
|
22
|
+
* @default false
|
23
|
+
*/
|
7
24
|
returnValue?: boolean;
|
8
25
|
}
|
9
26
|
|
10
27
|
function defaultStringify(value: unknown): string {
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
28
|
+
switch (typeof value) {
|
29
|
+
case 'undefined':
|
30
|
+
return 'undefined';
|
31
|
+
case 'string':
|
32
|
+
case 'number':
|
33
|
+
case 'boolean':
|
34
|
+
return JSON.stringify(value);
|
35
|
+
case 'bigint':
|
36
|
+
case 'symbol':
|
37
|
+
return value.toString();
|
38
|
+
case 'function':
|
39
|
+
return value.name + '()';
|
40
|
+
case 'object':
|
41
|
+
if (value === null) return 'null';
|
42
|
+
if (ArrayBuffer.isView(value)) {
|
43
|
+
const length = 'length' in value ? (value.length as number) : value.byteLength / (value.constructor as any).BYTES_PER_ELEMENT;
|
44
|
+
return `${value.constructor.name.replaceAll('Array', '').toLowerCase()}[${length}]`;
|
45
|
+
}
|
46
|
+
if (Array.isArray(value)) return `unknown[${value.length}]`;
|
47
|
+
try {
|
48
|
+
const json = JSON.stringify(value);
|
49
|
+
|
50
|
+
return json.length < 100 ? json : value.toString();
|
51
|
+
} catch {
|
52
|
+
return value.toString();
|
53
|
+
}
|
54
|
+
}
|
15
55
|
}
|
16
56
|
|
17
57
|
type LoggableDecoratorContext = Exclude<DecoratorContext, ClassFieldDecoratorContext>;
|
package/src/requests.ts
CHANGED
@@ -1,187 +1,73 @@
|
|
1
1
|
/* Utilities for `fetch` when using range requests. It also allows you to handle errors easier */
|
2
2
|
|
3
|
-
import
|
3
|
+
import * as cache from './cache.js';
|
4
4
|
|
5
|
-
|
6
|
-
|
7
|
-
export interface ResourceCacheOptions {
|
8
|
-
/**
|
9
|
-
* If true, use multiple buffers to cache a file.
|
10
|
-
* This is useful when working with small parts of large files,
|
11
|
-
* since we don't need to allocate a large buffer that is mostly unused
|
12
|
-
* @default true
|
13
|
-
*/
|
14
|
-
sparse?: boolean;
|
15
|
-
|
16
|
-
/**
|
17
|
-
* The threshold for whether to combine regions or not
|
18
|
-
* @see CacheRegion
|
19
|
-
* @default 0xfff // 4 KiB
|
20
|
-
*/
|
21
|
-
regionGapThreshold?: number;
|
22
|
-
}
|
23
|
-
|
24
|
-
export type CacheRange = { start: number; end: number };
|
25
|
-
|
26
|
-
export interface CacheRegion {
|
27
|
-
/** The region's offset from the start of the resource */
|
28
|
-
offset: number;
|
29
|
-
|
30
|
-
/** Ranges cached in this region. These are absolute! */
|
31
|
-
ranges: CacheRange[];
|
32
|
-
|
33
|
-
/** Data for this region */
|
34
|
-
data: Uint8Array;
|
35
|
-
}
|
36
|
-
|
37
|
-
/** The cache for a specific resource */
|
38
|
-
class ResourceCache {
|
39
|
-
/** Regions used to reduce unneeded allocations. Think of sparse arrays. */
|
40
|
-
public readonly regions: CacheRegion[] = [];
|
41
|
-
|
42
|
-
public constructor(
|
43
|
-
/** The resource URL */
|
44
|
-
public readonly url: string,
|
45
|
-
/** The full size of the resource */
|
46
|
-
public readonly size: number,
|
47
|
-
protected readonly options: ResourceCacheOptions
|
48
|
-
) {
|
49
|
-
options.sparse ??= true;
|
50
|
-
if (!options.sparse) this.regions.push({ offset: 0, data: new Uint8Array(size), ranges: [] });
|
51
|
-
|
52
|
-
requestsCache.set(url, this);
|
53
|
-
}
|
54
|
-
|
55
|
-
/** Combines adjacent regions and combines adjacent ranges within a region */
|
56
|
-
public collect(): void {
|
57
|
-
if (!this.options.sparse) return;
|
58
|
-
const { regionGapThreshold = 0xfff } = this.options;
|
59
|
-
|
60
|
-
for (let i = 0; i < this.regions.length - 1; ) {
|
61
|
-
const current = this.regions[i];
|
62
|
-
const next = this.regions[i + 1];
|
63
|
-
|
64
|
-
if (next.offset - (current.offset + current.data.byteLength) > regionGapThreshold) {
|
65
|
-
i++;
|
66
|
-
continue;
|
67
|
-
}
|
68
|
-
|
69
|
-
// Combine ranges
|
70
|
-
current.ranges.push(...next.ranges);
|
71
|
-
current.ranges.sort((a, b) => a.start - b.start);
|
72
|
-
|
73
|
-
// Combine overlapping/adjacent ranges
|
74
|
-
current.ranges = current.ranges.reduce((acc: CacheRange[], range) => {
|
75
|
-
if (!acc.length || acc.at(-1)!.end < range.start) {
|
76
|
-
acc.push(range);
|
77
|
-
} else {
|
78
|
-
acc.at(-1)!.end = Math.max(acc.at(-1)!.end, range.end);
|
79
|
-
}
|
80
|
-
return acc;
|
81
|
-
}, []);
|
82
|
-
|
83
|
-
// Extend buffer to include the new region
|
84
|
-
current.data = extendBuffer(current.data, next.offset + next.data.byteLength);
|
85
|
-
current.data.set(next.data, next.offset - current.offset);
|
86
|
-
|
87
|
-
// Remove the next region after merging
|
88
|
-
this.regions.splice(i + 1, 1);
|
89
|
-
}
|
90
|
-
}
|
91
|
-
|
92
|
-
/** Takes an initial range and finds the sub-ranges that are not in the cache */
|
93
|
-
public missing(start: number, end: number): CacheRange[] {
|
94
|
-
const missingRanges: CacheRange[] = [];
|
95
|
-
|
96
|
-
for (const region of this.regions) {
|
97
|
-
if (region.offset >= end) break;
|
5
|
+
// Compatibility
|
98
6
|
|
99
|
-
|
100
|
-
|
7
|
+
/** @deprecated Use `cache.Options` */
|
8
|
+
export type ResourceCacheOptions = cache.Options;
|
9
|
+
/** @deprecated Use `cache.Resource` */
|
10
|
+
export const ResourceCache = cache.Resource;
|
11
|
+
/** @deprecated Use `cache.Resource` */
|
12
|
+
export type ResourceCache = cache.Resource;
|
13
|
+
/** @deprecated Use `cache.Range` */
|
14
|
+
export type CacheRange = cache.Range;
|
15
|
+
/** @deprecated Use `cache.Options` */
|
16
|
+
export type CacheOptions = cache.Options;
|
101
17
|
|
102
|
-
|
103
|
-
|
104
|
-
if (range.start > start) {
|
105
|
-
missingRanges.push({ start, end: Math.min(range.start, end) });
|
106
|
-
}
|
107
|
-
|
108
|
-
// Adjust the current start if the region overlaps
|
109
|
-
if (range.end > start) start = Math.max(start, range.end);
|
110
|
-
|
111
|
-
if (start >= end) break;
|
112
|
-
}
|
113
|
-
|
114
|
-
if (start >= end) break;
|
115
|
-
}
|
116
|
-
|
117
|
-
// If there are still missing parts at the end
|
118
|
-
if (start < end) missingRanges.push({ start, end });
|
119
|
-
|
120
|
-
return missingRanges;
|
121
|
-
}
|
122
|
-
|
123
|
-
/** Get the region who's ranges include an offset */
|
124
|
-
public regionAt(offset: number): CacheRegion | undefined {
|
125
|
-
if (!this.regions.length) return;
|
126
|
-
|
127
|
-
for (const region of this.regions) {
|
128
|
-
if (region.offset > offset) break;
|
129
|
-
|
130
|
-
// Check if the offset is within this region
|
131
|
-
if (offset >= region.offset && offset < region.offset + region.data.byteLength) return region;
|
132
|
-
}
|
133
|
-
}
|
134
|
-
|
135
|
-
/** Add new data to the cache at given specified offset */
|
136
|
-
public add(data: Uint8Array, start: number, end: number = start + data.byteLength): this {
|
137
|
-
const region = this.regionAt(start);
|
138
|
-
|
139
|
-
if (region) {
|
140
|
-
region.data = extendBuffer(region.data, end);
|
141
|
-
region.ranges.push({ start, end });
|
142
|
-
region.ranges.sort((a, b) => a.start - b.start);
|
18
|
+
/* eslint-disable @typescript-eslint/only-throw-error */
|
143
19
|
|
144
|
-
|
145
|
-
|
20
|
+
/**
|
21
|
+
* @internal
|
22
|
+
*/
|
23
|
+
export const resourcesCache = new Map<string, cache.Resource | undefined>();
|
146
24
|
|
147
|
-
|
148
|
-
const newRegion: CacheRegion = { data, offset: start, ranges: [{ start, end }] };
|
149
|
-
const insertIndex = this.regions.findIndex(region => region.offset > start);
|
25
|
+
export type Issue = { tag: 'status'; response: Response } | { tag: 'buffer'; response: Response; message: string } | { tag: 'fetch' | 'size'; message: string } | Error;
|
150
26
|
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
this.regions.splice(insertIndex, 0, newRegion); // Insert before the first region with a greater offset
|
156
|
-
}
|
27
|
+
/**
|
28
|
+
* @deprecated Use `Issue`
|
29
|
+
*/
|
30
|
+
export type RequestError = Issue;
|
157
31
|
|
158
|
-
|
159
|
-
|
32
|
+
interface Fetched<TBodyOptional extends boolean> {
|
33
|
+
response: Response;
|
34
|
+
data: false extends TBodyOptional ? Uint8Array : Uint8Array | undefined;
|
160
35
|
}
|
161
36
|
|
162
|
-
const requestsCache = new Map<string, ResourceCache | null>();
|
163
|
-
|
164
|
-
export type RequestError = { tag: 'status'; response: Response } | { tag: 'buffer'; response: Response; message: string } | { tag: 'fetch' | 'size'; message: string } | Error;
|
165
|
-
|
166
37
|
/**
|
167
38
|
* Wraps `fetch`
|
168
39
|
* @throws RequestError
|
169
40
|
*/
|
170
|
-
async function
|
41
|
+
async function _fetch<const TBodyOptional extends boolean>(
|
42
|
+
input: RequestInfo,
|
43
|
+
init: RequestInit = {},
|
44
|
+
bodyOptional: TBodyOptional = false as TBodyOptional
|
45
|
+
): Promise<Fetched<TBodyOptional>> {
|
171
46
|
const response = await fetch(input, init).catch((error: Error) => {
|
172
|
-
throw { tag: 'fetch', message: error.message } satisfies
|
47
|
+
throw { tag: 'fetch', message: error.message } satisfies Issue;
|
173
48
|
});
|
174
49
|
|
175
|
-
if (!response.ok) throw { tag: 'status', response } satisfies
|
50
|
+
if (!response.ok) throw { tag: 'status', response } satisfies Issue;
|
176
51
|
|
177
|
-
const
|
178
|
-
|
52
|
+
const raw = await response.arrayBuffer().catch((error: Error) => {
|
53
|
+
if (bodyOptional) return;
|
54
|
+
throw { tag: 'buffer', response, message: error.message } satisfies Issue;
|
179
55
|
});
|
180
56
|
|
181
|
-
return { response, data: new Uint8Array(
|
57
|
+
return { response, data: raw ? new Uint8Array(raw) : undefined } as Fetched<TBodyOptional>;
|
58
|
+
}
|
59
|
+
|
60
|
+
export interface Options extends cache.Options {
|
61
|
+
/** Optionally provide a function for logging warnings */
|
62
|
+
warn?(message: string): unknown;
|
182
63
|
}
|
183
64
|
|
184
|
-
|
65
|
+
/**
|
66
|
+
* @deprecated Use `Options`
|
67
|
+
*/
|
68
|
+
export type RequestOptions = Options;
|
69
|
+
|
70
|
+
export interface GetOptions extends Options {
|
185
71
|
/**
|
186
72
|
* When using range requests,
|
187
73
|
* a HEAD request must normally be used to determine the full size of the resource.
|
@@ -194,22 +80,19 @@ export interface RequestOptions extends ResourceCacheOptions {
|
|
194
80
|
|
195
81
|
/** The end of the range */
|
196
82
|
end?: number;
|
197
|
-
|
198
|
-
/** Optionally provide a function for logging warnings */
|
199
|
-
warn?(message: string): unknown;
|
200
83
|
}
|
201
84
|
|
202
85
|
/**
|
203
86
|
* Make a GET request without worrying about ranges
|
204
87
|
* @throws RequestError
|
205
88
|
*/
|
206
|
-
export async function
|
89
|
+
export async function get(url: string, options: GetOptions, init: RequestInit = {}): Promise<Uint8Array> {
|
207
90
|
const req = new Request(url, init);
|
208
91
|
|
209
92
|
// Request no using ranges
|
210
93
|
if (typeof options.start != 'number' || typeof options.end != 'number') {
|
211
|
-
const { data } = await
|
212
|
-
new
|
94
|
+
const { data } = await _fetch(url, init);
|
95
|
+
new cache.Resource(url, data.byteLength, options, resourcesCache).add(data, 0);
|
213
96
|
return data;
|
214
97
|
}
|
215
98
|
|
@@ -220,47 +103,56 @@ export async function GET(url: string, options: RequestOptions, init: RequestIni
|
|
220
103
|
|
221
104
|
const { headers } = await fetch(req, { method: 'HEAD' });
|
222
105
|
const size = parseInt(headers.get('Content-Length') ?? '');
|
223
|
-
if (typeof size != 'number') throw { tag: 'size', message: 'Response is missing content-length header and no size was provided' } satisfies
|
106
|
+
if (typeof size != 'number') throw { tag: 'size', message: 'Response is missing content-length header and no size was provided' } satisfies Issue;
|
224
107
|
options.size = size;
|
225
108
|
}
|
226
109
|
|
227
110
|
const { size, start, end } = options;
|
228
|
-
const
|
111
|
+
const resource = resourcesCache.get(url) ?? new cache.Resource(url, size, options, resourcesCache);
|
229
112
|
|
230
113
|
req.headers.set('If-Range', new Date().toUTCString());
|
231
114
|
|
232
|
-
for (const { start: from, end: to } of
|
233
|
-
const { data, response } = await
|
115
|
+
for (const { start: from, end: to } of resource.missing(start, end)) {
|
116
|
+
const { data, response } = await _fetch(req, { headers: { Range: `bytes=${from}-${to}` } });
|
234
117
|
|
235
118
|
if (response.status == 206) {
|
236
|
-
|
119
|
+
resource.add(data, from);
|
237
120
|
continue;
|
238
121
|
}
|
239
122
|
|
240
123
|
// The first response doesn't have a "partial content" (206) status
|
241
124
|
options.warn?.(url + ': Remote does not support range requests with bytes. Falling back to full data.');
|
242
|
-
new
|
125
|
+
new cache.Resource(url, size, options, resourcesCache).add(data, 0);
|
243
126
|
return data.subarray(start, end);
|
244
127
|
}
|
245
128
|
|
246
129
|
// This ensures we get a single buffer with the entire requested range
|
247
|
-
|
130
|
+
resource.collect();
|
248
131
|
|
249
|
-
const region =
|
132
|
+
const region = resource.regionAt(start)!;
|
250
133
|
return region.data.subarray(start - region.offset, end - region.offset);
|
251
134
|
}
|
252
135
|
|
136
|
+
/**
|
137
|
+
* @deprecated Use `get`
|
138
|
+
*/
|
139
|
+
export const GET = get;
|
140
|
+
|
253
141
|
/**
|
254
142
|
* Synchronously gets a cached resource
|
255
143
|
* Assumes you pass valid start and end when using ranges
|
256
144
|
*/
|
257
|
-
export function getCached(url: string, options:
|
258
|
-
const cache =
|
145
|
+
export function getCached(url: string, options: GetOptions): { data?: Uint8Array; missing: cache.Range[] } {
|
146
|
+
const cache = resourcesCache.get(url);
|
259
147
|
|
260
148
|
/**
|
261
149
|
* @todo Make sure we have a size?
|
262
150
|
*/
|
263
|
-
if (!cache)
|
151
|
+
if (!cache) {
|
152
|
+
if (options.size) return { data: new Uint8Array(0), missing: [{ start: 0, end: options.size ?? 0 }] };
|
153
|
+
options.warn?.(url + ': Size not provided and cache is empty, can not determine missing range');
|
154
|
+
return { data: undefined, missing: [] };
|
155
|
+
}
|
264
156
|
|
265
157
|
const { start = 0, end = cache.size } = options;
|
266
158
|
|
@@ -285,3 +177,38 @@ export function getCached(url: string, options: RequestOptions): { data: Uint8Ar
|
|
285
177
|
|
286
178
|
return { data, missing: cache.missing(start, end) };
|
287
179
|
}
|
180
|
+
|
181
|
+
interface SetOptions extends Options {
|
182
|
+
/** The offset we are updating at */
|
183
|
+
offset?: number;
|
184
|
+
|
185
|
+
/** If a cache for the resource doesn't exist, this will be used as the full size */
|
186
|
+
size?: number;
|
187
|
+
}
|
188
|
+
|
189
|
+
/**
|
190
|
+
* Make a POST request to set (or create) data on the server and update the cache.
|
191
|
+
* @throws RequestError
|
192
|
+
*/
|
193
|
+
export async function set(url: string, data: Uint8Array, options: SetOptions, init: RequestInit = {}): Promise<void> {
|
194
|
+
if (!resourcesCache.has(url)) {
|
195
|
+
new cache.Resource(url, options.size ?? data.byteLength, options, resourcesCache);
|
196
|
+
}
|
197
|
+
|
198
|
+
const resource = resourcesCache.get(url)!;
|
199
|
+
|
200
|
+
const { offset = 0 } = options;
|
201
|
+
|
202
|
+
if (!options.cacheOnly) await _fetch(new Request(url, init), { method: 'POST' }, true);
|
203
|
+
|
204
|
+
resource.add(data, offset).collect();
|
205
|
+
}
|
206
|
+
|
207
|
+
/**
|
208
|
+
* Make a DELETE request to remove the resource from the server and clear it from the cache.
|
209
|
+
* @throws RequestError
|
210
|
+
*/
|
211
|
+
export async function remove(url: string, options: Options = {}, init: RequestInit = {}): Promise<void> {
|
212
|
+
if (!options.cacheOnly) await _fetch(new Request(url, init), { method: 'DELETE' }, true);
|
213
|
+
resourcesCache.delete(url);
|
214
|
+
}
|