weather-for-grown-ups 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cache/file-access-policy.d.ts +61 -0
- package/dist/cache/file-access-policy.js +156 -0
- package/dist/cache/file-access-policy.js.map +1 -0
- package/dist/cache/file-rate-limiter.d.ts +2 -8
- package/dist/cache/file-rate-limiter.js +11 -67
- package/dist/cache/file-rate-limiter.js.map +1 -1
- package/dist/cache/gefs-s3-subset-cache.d.ts +3 -1
- package/dist/cache/gefs-s3-subset-cache.js +57 -22
- package/dist/cache/gefs-s3-subset-cache.js.map +1 -1
- package/dist/cache/ifs-open-data-access-policy.d.ts +9 -0
- package/dist/cache/ifs-open-data-access-policy.js +22 -0
- package/dist/cache/ifs-open-data-access-policy.js.map +1 -0
- package/dist/cache/ifs-open-data-cache.d.ts +3 -1
- package/dist/cache/ifs-open-data-cache.js +43 -23
- package/dist/cache/ifs-open-data-cache.js.map +1 -1
- package/dist/cache/s3-subset-cache.d.ts +4 -1
- package/dist/cache/s3-subset-cache.js +79 -23
- package/dist/cache/s3-subset-cache.js.map +1 -1
- package/dist/catalog/catalog.d.ts +1 -1
- package/dist/cli/program.js +2 -1
- package/dist/cli/program.js.map +1 -1
- package/dist/cli/unified-atmosphere-command.js +28 -3
- package/dist/cli/unified-atmosphere-command.js.map +1 -1
- package/dist/core/history-forecast.js +5 -4
- package/dist/core/history-forecast.js.map +1 -1
- package/dist/core/history.js +2 -2
- package/dist/core/history.js.map +1 -1
- package/dist/core/ifs-area-summary.js +1 -1
- package/dist/core/ifs-area-summary.js.map +1 -1
- package/dist/core/ifs-ens-area-summary.js +1 -1
- package/dist/core/ifs-ens-area-summary.js.map +1 -1
- package/dist/core/ifs-ens-diagnostics.js +1 -1
- package/dist/core/ifs-ens-diagnostics.js.map +1 -1
- package/dist/core/ifs-ens-latest-run.d.ts +1 -0
- package/dist/core/ifs-ens-latest-run.js +12 -1
- package/dist/core/ifs-ens-latest-run.js.map +1 -1
- package/dist/core/ifs-ens-member-bundle.js +1 -1
- package/dist/core/ifs-ens-member-bundle.js.map +1 -1
- package/dist/core/ifs-latest-run.d.ts +1 -0
- package/dist/core/ifs-latest-run.js +12 -1
- package/dist/core/ifs-latest-run.js.map +1 -1
- package/dist/core/ifs-profile.js +1 -1
- package/dist/core/ifs-profile.js.map +1 -1
- package/dist/core/igra-observation.js +2 -2
- package/dist/core/igra-observation.js.map +1 -1
- package/dist/core/points-time-series.d.ts +3 -0
- package/dist/core/points-time-series.js +44 -9
- package/dist/core/points-time-series.js.map +1 -1
- package/dist/core/progress.d.ts +13 -0
- package/dist/core/progress.js +2 -0
- package/dist/core/progress.js.map +1 -0
- package/dist/core/time-series.d.ts +3 -0
- package/dist/core/time-series.js +45 -10
- package/dist/core/time-series.js.map +1 -1
- package/dist/core/unified-atmosphere-api.d.ts +2 -0
- package/dist/core/unified-atmosphere-api.js +10 -6
- package/dist/core/unified-atmosphere-api.js.map +1 -1
- package/dist/mcp-http-server.js +2 -1
- package/dist/mcp-http-server.js.map +1 -1
- package/dist/mcp-server.js +2 -1
- package/dist/mcp-server.js.map +1 -1
- package/dist/schema/unified-api.js +19 -2
- package/dist/schema/unified-api.js.map +1 -1
- package/dist/sources/http-retry.d.ts +15 -0
- package/dist/sources/http-retry.js +44 -0
- package/dist/sources/http-retry.js.map +1 -0
- package/dist/sources/ifs-open-data.d.ts +14 -2
- package/dist/sources/ifs-open-data.js +55 -41
- package/dist/sources/ifs-open-data.js.map +1 -1
- package/dist/sources/ncei-gfs-forecast-history.d.ts +4 -2
- package/dist/sources/ncei-gfs-forecast-history.js +47 -14
- package/dist/sources/ncei-gfs-forecast-history.js.map +1 -1
- package/dist/sources/ncei-gfs-history.d.ts +4 -2
- package/dist/sources/ncei-gfs-history.js +47 -14
- package/dist/sources/ncei-gfs-history.js.map +1 -1
- package/dist/sources/ncei-igra.d.ts +4 -2
- package/dist/sources/ncei-igra.js +81 -14
- package/dist/sources/ncei-igra.js.map +1 -1
- package/dist/sources/rda-gfs-forecast-history.d.ts +4 -2
- package/dist/sources/rda-gfs-forecast-history.js +25 -9
- package/dist/sources/rda-gfs-forecast-history.js.map +1 -1
- package/dist/version.d.ts +1 -0
- package/dist/version.js +10 -0
- package/dist/version.js.map +1 -0
- package/docs/ARCHITECTURE.md +5 -5
- package/docs/UNIFIED_API.md +2 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -260,12 +260,12 @@ The HTTP server defaults to `127.0.0.1:3000`, serves MCP at `/mcp`, and exposes
|
|
|
260
260
|
|
|
261
261
|
WFG selects only the GRIB messages needed for a query, caches immutable upstream slices, decodes locally, and performs physical transforms and aggregation in the TypeScript core.
|
|
262
262
|
|
|
263
|
-
- GFS
|
|
263
|
+
- GFS source routing is automatic by default: point/profile, time-series, multi-point, transect, and run-comparison access uses NOAA AWS Open Data selected-message byte ranges; bounded area queries use NOAA NOMADS geographic subsetting. `--source` remains an explicit override where that geometry supports it, and result provenance reports the resolved backend.
|
|
264
264
|
- GEFS uses NOAA AWS Open Data `.idx` inventories and byte-range access per member. Pressure/mixed selections use `pgrb2a` 0.5°; eligible field-only selections use `pgrb2s` 0.25° through `f240`.
|
|
265
265
|
- The npm package ships with a GRIB2 decoder; native `wgrib2` remains an optional compatibility/debug path.
|
|
266
266
|
- Historical GFS analysis uses NOAA NCEI THREDDS/NCSS: grid-as-point for point/profile operations and native bbox/grid subsets for area statistics.
|
|
267
267
|
- Historical GFS forecasts keep the public `gfs` identity and route by grid: 0.25° uses NCAR/GDEX d084001 through THREDDS/NCSS, while 0.5° uses NOAA NCEI Grid 4 through THREDDS/NCSS. Direct online availability varies; older Grid 4 files may require NCEI HAS retrieval.
|
|
268
|
-
-
|
|
268
|
+
- Upstream etiquette is provider-specific: NOMADS keeps an 11-second cross-process courtesy interval, NCEI THREDDS/NCSS is bounded to 2 concurrent requests, NCAR/GDEX to 4, and IGRA downloads to 4. NOAA AWS and ECMWF cloud/direct access are bounded separately without inheriting the NOMADS delay. Transient 429/5xx responses use exponential backoff with jitter and honor `Retry-After`.
|
|
269
269
|
|
|
270
270
|
## Documentation
|
|
271
271
|
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export declare const DEFAULT_ACCESS_POLICY_STALE_LOCK_MS = 120000;
|
|
2
|
+
export interface UpstreamAccessPolicy {
|
|
3
|
+
run<T>(operation: () => Promise<T>): Promise<T>;
|
|
4
|
+
}
|
|
5
|
+
export interface UpstreamAccessPolicyDefinition {
|
|
6
|
+
id: string;
|
|
7
|
+
maxConcurrency: number;
|
|
8
|
+
minIntervalMs: number;
|
|
9
|
+
staleLockMs?: number;
|
|
10
|
+
}
|
|
11
|
+
export declare const UPSTREAM_ACCESS_POLICIES: {
|
|
12
|
+
readonly nomads: {
|
|
13
|
+
readonly id: "nomads";
|
|
14
|
+
readonly maxConcurrency: 1;
|
|
15
|
+
readonly minIntervalMs: 11000;
|
|
16
|
+
};
|
|
17
|
+
readonly noaaAws: {
|
|
18
|
+
readonly id: "noaa-aws";
|
|
19
|
+
readonly maxConcurrency: 8;
|
|
20
|
+
readonly minIntervalMs: 0;
|
|
21
|
+
};
|
|
22
|
+
readonly ecmwfCloud: {
|
|
23
|
+
readonly id: "ecmwf-cloud";
|
|
24
|
+
readonly maxConcurrency: 8;
|
|
25
|
+
readonly minIntervalMs: 0;
|
|
26
|
+
};
|
|
27
|
+
readonly ecmwfDirect: {
|
|
28
|
+
readonly id: "ecmwf-direct";
|
|
29
|
+
readonly maxConcurrency: 4;
|
|
30
|
+
readonly minIntervalMs: 0;
|
|
31
|
+
};
|
|
32
|
+
readonly nceiThredds: {
|
|
33
|
+
readonly id: "ncei-thredds";
|
|
34
|
+
readonly maxConcurrency: 2;
|
|
35
|
+
readonly minIntervalMs: 0;
|
|
36
|
+
};
|
|
37
|
+
readonly gdex: {
|
|
38
|
+
readonly id: "gdex";
|
|
39
|
+
readonly maxConcurrency: 4;
|
|
40
|
+
readonly minIntervalMs: 0;
|
|
41
|
+
};
|
|
42
|
+
readonly nceiIgra: {
|
|
43
|
+
readonly id: "ncei-igra";
|
|
44
|
+
readonly maxConcurrency: 4;
|
|
45
|
+
readonly minIntervalMs: 0;
|
|
46
|
+
};
|
|
47
|
+
};
|
|
48
|
+
export declare class FileAccessPolicy implements UpstreamAccessPolicy {
|
|
49
|
+
private readonly rootDir;
|
|
50
|
+
private readonly definition;
|
|
51
|
+
private readonly pollMs;
|
|
52
|
+
private readonly staleLockMs;
|
|
53
|
+
constructor(rootDir: string, definition: UpstreamAccessPolicyDefinition, pollMs?: number);
|
|
54
|
+
run<T>(operation: () => Promise<T>): Promise<T>;
|
|
55
|
+
private startHeartbeat;
|
|
56
|
+
private acquireSlot;
|
|
57
|
+
private slotPath;
|
|
58
|
+
private statePath;
|
|
59
|
+
private readState;
|
|
60
|
+
}
|
|
61
|
+
export declare function withLegacyCooldown(definition: UpstreamAccessPolicyDefinition, cooldownMs: number | undefined): UpstreamAccessPolicyDefinition;
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { mkdir, readFile, rm, stat, utimes, writeFile } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
4
|
+
export const DEFAULT_ACCESS_POLICY_STALE_LOCK_MS = 120_000;
|
|
5
|
+
const DEFAULT_POLL_MS = 100;
|
|
6
|
+
export const UPSTREAM_ACCESS_POLICIES = {
|
|
7
|
+
nomads: {
|
|
8
|
+
id: "nomads",
|
|
9
|
+
maxConcurrency: 1,
|
|
10
|
+
minIntervalMs: 11_000,
|
|
11
|
+
},
|
|
12
|
+
noaaAws: {
|
|
13
|
+
id: "noaa-aws",
|
|
14
|
+
maxConcurrency: 8,
|
|
15
|
+
minIntervalMs: 0,
|
|
16
|
+
},
|
|
17
|
+
ecmwfCloud: {
|
|
18
|
+
id: "ecmwf-cloud",
|
|
19
|
+
maxConcurrency: 8,
|
|
20
|
+
minIntervalMs: 0,
|
|
21
|
+
},
|
|
22
|
+
ecmwfDirect: {
|
|
23
|
+
id: "ecmwf-direct",
|
|
24
|
+
maxConcurrency: 4,
|
|
25
|
+
minIntervalMs: 0,
|
|
26
|
+
},
|
|
27
|
+
nceiThredds: {
|
|
28
|
+
id: "ncei-thredds",
|
|
29
|
+
maxConcurrency: 2,
|
|
30
|
+
minIntervalMs: 0,
|
|
31
|
+
},
|
|
32
|
+
gdex: {
|
|
33
|
+
id: "gdex",
|
|
34
|
+
maxConcurrency: 4,
|
|
35
|
+
minIntervalMs: 0,
|
|
36
|
+
},
|
|
37
|
+
nceiIgra: {
|
|
38
|
+
id: "ncei-igra",
|
|
39
|
+
maxConcurrency: 4,
|
|
40
|
+
minIntervalMs: 0,
|
|
41
|
+
},
|
|
42
|
+
};
|
|
43
|
+
export class FileAccessPolicy {
|
|
44
|
+
rootDir;
|
|
45
|
+
definition;
|
|
46
|
+
pollMs;
|
|
47
|
+
staleLockMs;
|
|
48
|
+
constructor(rootDir, definition, pollMs = DEFAULT_POLL_MS) {
|
|
49
|
+
this.rootDir = rootDir;
|
|
50
|
+
this.definition = definition;
|
|
51
|
+
this.pollMs = pollMs;
|
|
52
|
+
if (!definition.id.match(/^[a-z0-9-]+$/)) {
|
|
53
|
+
throw new Error("access policy id must contain only lowercase letters, digits, and hyphens");
|
|
54
|
+
}
|
|
55
|
+
if (!Number.isInteger(definition.maxConcurrency) || definition.maxConcurrency < 1) {
|
|
56
|
+
throw new Error("access policy maxConcurrency must be a positive integer");
|
|
57
|
+
}
|
|
58
|
+
if (!Number.isFinite(definition.minIntervalMs) || definition.minIntervalMs < 0) {
|
|
59
|
+
throw new Error("access policy minIntervalMs must be non-negative");
|
|
60
|
+
}
|
|
61
|
+
if (definition.minIntervalMs > 0 && definition.maxConcurrency !== 1) {
|
|
62
|
+
throw new Error("access policies with a minimum interval must use maxConcurrency=1");
|
|
63
|
+
}
|
|
64
|
+
this.staleLockMs = definition.staleLockMs ?? DEFAULT_ACCESS_POLICY_STALE_LOCK_MS;
|
|
65
|
+
}
|
|
66
|
+
async run(operation) {
|
|
67
|
+
await mkdir(this.rootDir, { recursive: true });
|
|
68
|
+
const slot = await this.acquireSlot();
|
|
69
|
+
const stopHeartbeat = this.startHeartbeat(slot);
|
|
70
|
+
try {
|
|
71
|
+
if (this.definition.minIntervalMs > 0) {
|
|
72
|
+
const state = await this.readState();
|
|
73
|
+
const waitMs = Math.max(0, state.lastRequestCompletedAt + this.definition.minIntervalMs - Date.now());
|
|
74
|
+
if (waitMs > 0)
|
|
75
|
+
await sleep(waitMs);
|
|
76
|
+
}
|
|
77
|
+
try {
|
|
78
|
+
return await operation();
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
if (this.definition.minIntervalMs > 0) {
|
|
82
|
+
await writeFile(this.statePath(), JSON.stringify({ lastRequestCompletedAt: Date.now() }), "utf8");
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
finally {
|
|
87
|
+
stopHeartbeat();
|
|
88
|
+
await rm(this.slotPath(slot), { recursive: true, force: true });
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
startHeartbeat(slot) {
|
|
92
|
+
const path = this.slotPath(slot);
|
|
93
|
+
const intervalMs = Math.max(1, Math.floor(this.staleLockMs / 3));
|
|
94
|
+
const timer = setInterval(() => {
|
|
95
|
+
const now = new Date();
|
|
96
|
+
void utimes(path, now, now).catch(() => undefined);
|
|
97
|
+
}, intervalMs);
|
|
98
|
+
timer.unref();
|
|
99
|
+
return () => clearInterval(timer);
|
|
100
|
+
}
|
|
101
|
+
async acquireSlot() {
|
|
102
|
+
for (;;) {
|
|
103
|
+
for (let slot = 0; slot < this.definition.maxConcurrency; slot += 1) {
|
|
104
|
+
const path = this.slotPath(slot);
|
|
105
|
+
try {
|
|
106
|
+
await mkdir(path);
|
|
107
|
+
return slot;
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
if (!isAlreadyExists(error))
|
|
111
|
+
throw error;
|
|
112
|
+
try {
|
|
113
|
+
const lockStat = await stat(path);
|
|
114
|
+
if (Date.now() - lockStat.mtimeMs > this.staleLockMs) {
|
|
115
|
+
await rm(path, { recursive: true, force: true });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
catch {
|
|
119
|
+
// Another process may have released the slot between checks.
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
await sleep(this.pollMs);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
slotPath(slot) {
|
|
127
|
+
if (this.definition.maxConcurrency === 1) {
|
|
128
|
+
return join(this.rootDir, `${this.definition.id}.lock`);
|
|
129
|
+
}
|
|
130
|
+
return join(this.rootDir, `${this.definition.id}.slot-${slot}.lock`);
|
|
131
|
+
}
|
|
132
|
+
statePath() {
|
|
133
|
+
return join(this.rootDir, `${this.definition.id}-state.json`);
|
|
134
|
+
}
|
|
135
|
+
async readState() {
|
|
136
|
+
try {
|
|
137
|
+
return JSON.parse(await readFile(this.statePath(), "utf8"));
|
|
138
|
+
}
|
|
139
|
+
catch {
|
|
140
|
+
return { lastRequestCompletedAt: 0 };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
export function withLegacyCooldown(definition, cooldownMs) {
|
|
145
|
+
if (cooldownMs === undefined)
|
|
146
|
+
return definition;
|
|
147
|
+
return {
|
|
148
|
+
...definition,
|
|
149
|
+
maxConcurrency: 1,
|
|
150
|
+
minIntervalMs: cooldownMs,
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function isAlreadyExists(error) {
|
|
154
|
+
return error instanceof Error && "code" in error && error.code === "EEXIST";
|
|
155
|
+
}
|
|
156
|
+
//# sourceMappingURL=file-access-policy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-access-policy.js","sourceRoot":"","sources":["../../src/cache/file-access-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAChF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,MAAM,KAAK,GAAG,CAAC,EAAU,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAEhF,MAAM,CAAC,MAAM,mCAAmC,GAAG,OAAO,CAAC;AAC3D,MAAM,eAAe,GAAG,GAAG,CAAC;AAa5B,MAAM,CAAC,MAAM,wBAAwB,GAAG;IACtC,MAAM,EAAE;QACN,EAAE,EAAE,QAAQ;QACZ,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,MAAM;KACtB;IACD,OAAO,EAAE;QACP,EAAE,EAAE,UAAU;QACd,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB;IACD,UAAU,EAAE;QACV,EAAE,EAAE,aAAa;QACjB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB;IACD,WAAW,EAAE;QACX,EAAE,EAAE,cAAc;QAClB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB;IACD,WAAW,EAAE;QACX,EAAE,EAAE,cAAc;QAClB,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB;IACD,IAAI,EAAE;QACJ,EAAE,EAAE,MAAM;QACV,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB;IACD,QAAQ,EAAE;QACR,EAAE,EAAE,WAAW;QACf,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,CAAC;KACjB;CACgE,CAAC;AAMpE,MAAM,OAAO,gBAAgB;IAIR;IACA;IACA;IALF,WAAW,CAAS;IAErC,YACmB,OAAe,EACf,UAA0C,EAC1C,SAAS,eAAe;QAFxB,YAAO,GAAP,OAAO,CAAQ;QACf,eAAU,GAAV,UAAU,CAAgC;QAC1C,WAAM,GAAN,MAAM,CAAkB;QAEzC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,cAAc,CAAC,IAAI,UAAU,CAAC,cAAc,GAAG,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,UAAU,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;YAC/E,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAC;QACtE,CAAC;QACD,IAAI,UAAU,CAAC,aAAa,GAAG,CAAC,IAAI,UAAU,CAAC,cAAc,KAAK,CAAC,EAAE,CAAC;YACpE,MAAM,IAAI,KAAK,CAAC,mEAAmE,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC,WAAW,IAAI,mCAAmC,CAAC;IACnF,CAAC;IAED,KAAK,CAAC,GAAG,CAAI,SAA2B;QACtC,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;QAEhD,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,EAAE,CAAC;gBACrC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CACrB,CAAC,EACD,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAC1E,CAAC;gBACF,IAAI,MAAM,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC;YACtC,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,MAAM,SAAS,EAAE,CAAC;YAC3B,CAAC;oBAAS,CAAC;gBACT,IAAI,IAAI,CAAC,UAAU,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;oBACtC,MAAM,SAAS,CACb,IAAI,CAAC,SAAS,EAAE,EAChB,IAAI,CAAC,SAAS,CAAC,EAAE,sBAAsB,EAAE,IAAI,CAAC,GAAG,EAAE,EAAkB,CAAC,EACtE,MAAM,CACP,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,aAAa,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,IAAY;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACjC,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE;YAC7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,KAAK,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACrD,CAAC,EAAE,UAAU,CAAC,CAAC;QACf,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,SAAS,CAAC;YACR,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;gBACpE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACjC,IAAI,CAAC;oBACH,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;oBAClB,OAAO,IAAI,CAAC;gBACd,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC;wBAAE,MAAM,KAAK,CAAC;oBAEzC,IAAI,CAAC;wBACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;wBAClC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;4BACrD,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;wBACnD,CAAC;oBACH,CAAC;oBAAC,MAAM,CAAC;wBACP,6DAA6D;oBAC/D,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,IAAY;QAC3B,IAAI,IAAI,CAAC,UAAU,CAAC,cAAc,KAAK,CAAC,EAAE,CAAC;YACzC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1D,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,SAAS,IAAI,OAAO,CAAC,CAAC;IACvE,CAAC;IAEO,SAAS;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,EAAE,aAAa,CAAC,CAAC;IAChE,CAAC;IAEO,KAAK,CAAC,SAAS;QACrB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,CAAC,CAAU,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,sBAAsB,EAAE,CAAC,EAAE,CAAC;QACvC,CAAC;IACH,CAAC;CACF;AAED,MAAM,UAAU,kBAAkB,CAChC,UAA0C,EAC1C,UAA8B;IAE9B,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,UAAU,CAAC;IAChD,OAAO;QACL,GAAG,UAAU;QACb,cAAc,EAAE,CAAC;QACjB,aAAa,EAAE,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,OAAO,KAAK,YAAY,KAAK,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC;AAC9E,CAAC"}
|
|
@@ -1,13 +1,7 @@
|
|
|
1
|
-
export declare const DEFAULT_NOMADS_COOLDOWN_MS
|
|
1
|
+
export declare const DEFAULT_NOMADS_COOLDOWN_MS: 11000;
|
|
2
2
|
export declare const DEFAULT_STALE_LOCK_MS = 120000;
|
|
3
3
|
export declare class FileRateLimiter {
|
|
4
|
-
private readonly
|
|
5
|
-
private readonly cooldownMs;
|
|
6
|
-
private readonly staleLockMs;
|
|
7
|
-
private readonly lockDir;
|
|
8
|
-
private readonly statePath;
|
|
4
|
+
private readonly delegate;
|
|
9
5
|
constructor(rootDir: string, cooldownMs?: number, staleLockMs?: number);
|
|
10
6
|
run<T>(operation: () => Promise<T>): Promise<T>;
|
|
11
|
-
private acquire;
|
|
12
|
-
private readState;
|
|
13
7
|
}
|
|
@@ -1,73 +1,17 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
const
|
|
4
|
-
export const DEFAULT_NOMADS_COOLDOWN_MS = 11_000;
|
|
5
|
-
export const DEFAULT_STALE_LOCK_MS = 120_000;
|
|
1
|
+
import { DEFAULT_ACCESS_POLICY_STALE_LOCK_MS, FileAccessPolicy, UPSTREAM_ACCESS_POLICIES, } from "./file-access-policy.js";
|
|
2
|
+
export const DEFAULT_NOMADS_COOLDOWN_MS = UPSTREAM_ACCESS_POLICIES.nomads.minIntervalMs;
|
|
3
|
+
export const DEFAULT_STALE_LOCK_MS = DEFAULT_ACCESS_POLICY_STALE_LOCK_MS;
|
|
6
4
|
export class FileRateLimiter {
|
|
7
|
-
|
|
8
|
-
cooldownMs;
|
|
9
|
-
staleLockMs;
|
|
10
|
-
lockDir;
|
|
11
|
-
statePath;
|
|
5
|
+
delegate;
|
|
12
6
|
constructor(rootDir, cooldownMs = DEFAULT_NOMADS_COOLDOWN_MS, staleLockMs = DEFAULT_STALE_LOCK_MS) {
|
|
13
|
-
this.
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
7
|
+
this.delegate = new FileAccessPolicy(rootDir, {
|
|
8
|
+
...UPSTREAM_ACCESS_POLICIES.nomads,
|
|
9
|
+
minIntervalMs: cooldownMs,
|
|
10
|
+
staleLockMs,
|
|
11
|
+
});
|
|
18
12
|
}
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
await this.acquire();
|
|
22
|
-
try {
|
|
23
|
-
const state = await this.readState();
|
|
24
|
-
const waitMs = Math.max(0, state.lastRequestCompletedAt + this.cooldownMs - Date.now());
|
|
25
|
-
if (waitMs > 0)
|
|
26
|
-
await sleep(waitMs);
|
|
27
|
-
try {
|
|
28
|
-
return await operation();
|
|
29
|
-
}
|
|
30
|
-
finally {
|
|
31
|
-
await writeFile(this.statePath, JSON.stringify({ lastRequestCompletedAt: Date.now() }), "utf8");
|
|
32
|
-
}
|
|
33
|
-
}
|
|
34
|
-
finally {
|
|
35
|
-
await rm(this.lockDir, { recursive: true, force: true });
|
|
36
|
-
}
|
|
13
|
+
run(operation) {
|
|
14
|
+
return this.delegate.run(operation);
|
|
37
15
|
}
|
|
38
|
-
async acquire() {
|
|
39
|
-
for (;;) {
|
|
40
|
-
try {
|
|
41
|
-
await mkdir(this.lockDir);
|
|
42
|
-
return;
|
|
43
|
-
}
|
|
44
|
-
catch (error) {
|
|
45
|
-
if (!isAlreadyExists(error))
|
|
46
|
-
throw error;
|
|
47
|
-
try {
|
|
48
|
-
const lockStat = await stat(this.lockDir);
|
|
49
|
-
if (Date.now() - lockStat.mtimeMs > this.staleLockMs) {
|
|
50
|
-
await rm(this.lockDir, { recursive: true, force: true });
|
|
51
|
-
continue;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
catch {
|
|
55
|
-
// Another process may have released the lock between checks.
|
|
56
|
-
}
|
|
57
|
-
await sleep(100);
|
|
58
|
-
}
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
async readState() {
|
|
62
|
-
try {
|
|
63
|
-
return JSON.parse(await readFile(this.statePath, "utf8"));
|
|
64
|
-
}
|
|
65
|
-
catch {
|
|
66
|
-
return { lastRequestCompletedAt: 0 };
|
|
67
|
-
}
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
function isAlreadyExists(error) {
|
|
71
|
-
return error instanceof Error && "code" in error && error.code === "EEXIST";
|
|
72
16
|
}
|
|
73
17
|
//# sourceMappingURL=file-rate-limiter.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-rate-limiter.js","sourceRoot":"","sources":["../../src/cache/file-rate-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"file-rate-limiter.js","sourceRoot":"","sources":["../../src/cache/file-rate-limiter.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mCAAmC,EACnC,gBAAgB,EAChB,wBAAwB,GACzB,MAAM,yBAAyB,CAAC;AAEjC,MAAM,CAAC,MAAM,0BAA0B,GAAG,wBAAwB,CAAC,MAAM,CAAC,aAAa,CAAC;AACxF,MAAM,CAAC,MAAM,qBAAqB,GAAG,mCAAmC,CAAC;AAEzE,MAAM,OAAO,eAAe;IACT,QAAQ,CAAmB;IAE5C,YACE,OAAe,EACf,aAAqB,0BAA0B,EAC/C,cAAsB,qBAAqB;QAE3C,IAAI,CAAC,QAAQ,GAAG,IAAI,gBAAgB,CAAC,OAAO,EAAE;YAC5C,GAAG,wBAAwB,CAAC,MAAM;YAClC,aAAa,EAAE,UAAU;YACzB,WAAW;SACZ,CAAC,CAAC;IACL,CAAC;IAED,GAAG,CAAI,SAA2B;QAChC,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;CACF"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type UpstreamAccessPolicy } from "./file-access-policy.js";
|
|
1
2
|
import type { GefsMember } from "../catalog/gefs.js";
|
|
2
3
|
import type { GfsCode } from "../catalog/variables.js";
|
|
3
4
|
import { type NonIsobaricGribSelector } from "../grib/index.js";
|
|
@@ -31,8 +32,9 @@ export interface GefsMemberSelectionSource {
|
|
|
31
32
|
export declare class GefsS3SubsetCache implements GefsMemberSource, GefsMemberSelectionSource {
|
|
32
33
|
private readonly rootDir;
|
|
33
34
|
private readonly fetchFn;
|
|
35
|
+
private readonly accessPolicy;
|
|
34
36
|
private readonly inFlight;
|
|
35
|
-
constructor(rootDir: string, fetchFn?: typeof fetch);
|
|
37
|
+
constructor(rootDir: string, fetchFn?: typeof fetch, accessPolicy?: UpstreamAccessPolicy);
|
|
36
38
|
fetch(request: GefsMemberDataRequest): Promise<GefsSubsetFile>;
|
|
37
39
|
fetchSelection(request: GefsMemberSelectionDataRequest): Promise<GefsSubsetFile>;
|
|
38
40
|
private fetchCached;
|
|
@@ -1,15 +1,19 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { access, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { join } from "node:path";
|
|
4
|
+
import { FileAccessPolicy, UPSTREAM_ACCESS_POLICIES, } from "./file-access-policy.js";
|
|
4
5
|
import { mergeByteRanges, parseGribIndex, selectNonIsobaricByteRanges, selectPressureByteRanges, } from "../grib/index.js";
|
|
6
|
+
import { DEFAULT_HTTP_RETRY_MAX_ATTEMPTS, isRetryableHttpStatus, waitBeforeHttpRetry, } from "../sources/http-retry.js";
|
|
5
7
|
import { buildGefsS3ForecastIndexUrl, buildGefsS3ForecastUrl, gefsAtmosProductForSelection, } from "../sources/gefs-s3.js";
|
|
6
8
|
export class GefsS3SubsetCache {
|
|
7
9
|
rootDir;
|
|
8
10
|
fetchFn;
|
|
11
|
+
accessPolicy;
|
|
9
12
|
inFlight = new Map();
|
|
10
|
-
constructor(rootDir, fetchFn = globalThis.fetch) {
|
|
13
|
+
constructor(rootDir, fetchFn = globalThis.fetch, accessPolicy = new FileAccessPolicy(join(rootDir, "access-state"), UPSTREAM_ACCESS_POLICIES.noaaAws)) {
|
|
11
14
|
this.rootDir = rootDir;
|
|
12
15
|
this.fetchFn = fetchFn;
|
|
16
|
+
this.accessPolicy = accessPolicy;
|
|
13
17
|
}
|
|
14
18
|
async fetch(request) {
|
|
15
19
|
return this.fetchCached(subsetKey(request), request, [request.variableCode], [request.pressureLevelHpa], [], "pgrb2a_0p50");
|
|
@@ -76,32 +80,63 @@ export class GefsS3SubsetCache {
|
|
|
76
80
|
catch {
|
|
77
81
|
// GEFS forecast files are immutable after publication, so index responses cache forever locally.
|
|
78
82
|
}
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
83
|
+
for (let attempt = 1; attempt <= DEFAULT_HTTP_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
|
84
|
+
const result = await this.accessPolicy.run(async () => {
|
|
85
|
+
const response = await this.fetchFn(url, {
|
|
86
|
+
headers: { "user-agent": "weather-for-grown-ups/0.1" },
|
|
87
|
+
});
|
|
88
|
+
return {
|
|
89
|
+
status: response.status,
|
|
90
|
+
statusText: response.statusText,
|
|
91
|
+
retryAfter: response.headers.get("retry-after"),
|
|
92
|
+
text: response.ok ? await response.text() : undefined,
|
|
93
|
+
};
|
|
94
|
+
});
|
|
95
|
+
if (isRetryableHttpStatus(result.status) && attempt < DEFAULT_HTTP_RETRY_MAX_ATTEMPTS) {
|
|
96
|
+
await waitBeforeHttpRetry(attempt, result.retryAfter);
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (result.status < 200 || result.status >= 300 || result.text === undefined) {
|
|
100
|
+
throw new Error(`NOAA GEFS AWS index request failed: HTTP ${result.status} ${result.statusText}`);
|
|
101
|
+
}
|
|
102
|
+
await writeFile(path, result.text, "utf8");
|
|
103
|
+
return result.text;
|
|
84
104
|
}
|
|
85
|
-
|
|
86
|
-
await writeFile(path, text, "utf8");
|
|
87
|
-
return text;
|
|
105
|
+
throw new Error("NOAA GEFS AWS index retry loop exhausted unexpectedly");
|
|
88
106
|
}
|
|
89
107
|
async fetchRange(url, range) {
|
|
90
108
|
const rangeValue = `bytes=${range.start}-${range.end ?? ""}`;
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
109
|
+
for (let attempt = 1; attempt <= DEFAULT_HTTP_RETRY_MAX_ATTEMPTS; attempt += 1) {
|
|
110
|
+
const result = await this.accessPolicy.run(async () => {
|
|
111
|
+
const response = await this.fetchFn(url, {
|
|
112
|
+
headers: {
|
|
113
|
+
range: rangeValue,
|
|
114
|
+
"user-agent": "weather-for-grown-ups/0.1",
|
|
115
|
+
},
|
|
116
|
+
});
|
|
117
|
+
return {
|
|
118
|
+
status: response.status,
|
|
119
|
+
statusText: response.statusText,
|
|
120
|
+
retryAfter: response.headers.get("retry-after"),
|
|
121
|
+
bytes: response.status === 206
|
|
122
|
+
? new Uint8Array(await response.arrayBuffer())
|
|
123
|
+
: undefined,
|
|
124
|
+
};
|
|
125
|
+
});
|
|
126
|
+
if (isRetryableHttpStatus(result.status) && attempt < DEFAULT_HTTP_RETRY_MAX_ATTEMPTS) {
|
|
127
|
+
await waitBeforeHttpRetry(attempt, result.retryAfter);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (result.status !== 206 || result.bytes === undefined) {
|
|
131
|
+
throw new Error(`NOAA GEFS AWS range request failed: HTTP ${result.status} ${result.statusText}`);
|
|
132
|
+
}
|
|
133
|
+
if (result.bytes.length < 4
|
|
134
|
+
|| new TextDecoder().decode(result.bytes.slice(0, 4)) !== "GRIB") {
|
|
135
|
+
throw new Error(`NOAA GEFS AWS range did not start with a GRIB message (${rangeValue})`);
|
|
136
|
+
}
|
|
137
|
+
return result.bytes;
|
|
99
138
|
}
|
|
100
|
-
|
|
101
|
-
if (bytes.length < 4 || new TextDecoder().decode(bytes.slice(0, 4)) !== "GRIB") {
|
|
102
|
-
throw new Error(`NOAA GEFS AWS range did not start with a GRIB message (${rangeValue})`);
|
|
103
|
-
}
|
|
104
|
-
return bytes;
|
|
139
|
+
throw new Error("NOAA GEFS AWS range retry loop exhausted unexpectedly");
|
|
105
140
|
}
|
|
106
141
|
}
|
|
107
142
|
function subsetKey(request) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gefs-s3-subset-cache.js","sourceRoot":"","sources":["../../src/cache/gefs-s3-subset-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EACL,eAAe,EACf,cAAc,EACd,2BAA2B,EAC3B,wBAAwB,GAGzB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,4BAA4B,GAE7B,MAAM,uBAAuB,CAAC;AAiC/B,MAAM,OAAO,iBAAiB;IAIT;IACA;
|
|
1
|
+
{"version":3,"file":"gefs-s3-subset-cache.js","sourceRoot":"","sources":["../../src/cache/gefs-s3-subset-cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAClF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,gBAAgB,EAChB,wBAAwB,GAEzB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,EACL,eAAe,EACf,cAAc,EACd,2BAA2B,EAC3B,wBAAwB,GAGzB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,+BAA+B,EAC/B,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,4BAA4B,GAE7B,MAAM,uBAAuB,CAAC;AAiC/B,MAAM,OAAO,iBAAiB;IAIT;IACA;IACA;IALF,QAAQ,GAAG,IAAI,GAAG,EAAmC,CAAC;IAEvE,YACmB,OAAe,EACf,UAAwB,UAAU,CAAC,KAAK,EACxC,eAAqC,IAAI,gBAAgB,CACxE,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAC7B,wBAAwB,CAAC,OAAO,CACjC;QALgB,YAAO,GAAP,OAAO,CAAQ;QACf,YAAO,GAAP,OAAO,CAAiC;QACxC,iBAAY,GAAZ,YAAY,CAG5B;IACA,CAAC;IAEJ,KAAK,CAAC,KAAK,CAAC,OAA8B;QACxC,OAAO,IAAI,CAAC,WAAW,CACrB,SAAS,CAAC,OAAO,CAAC,EAClB,OAAO,EACP,CAAC,OAAO,CAAC,YAAY,CAAC,EACtB,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAC1B,EAAE,EACF,aAAa,CACd,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,OAAuC;QAC1D,MAAM,aAAa,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjE,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,4BAA4B,CAC7D,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,CAAC,MAAM,GAAG,CAAC,EACxD,OAAO,CAAC,YAAY,CACrB,CAAC;QACF,OAAO,IAAI,CAAC,WAAW,CACrB,kBAAkB,CAAC,EAAE,GAAG,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EACrF,OAAO,EACP,aAAa,EACb,iBAAiB,EACjB,MAAM,EACN,OAAO,CACR,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,GAAW,EACX,OAAuE,EACvE,aAAwB,EACxB,iBAA2B,EAC3B,MAAiC,EACjC,OAAyB;QAEzB,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,QAAQ,CAAC,CAAC;QAChD,IAAI,MAAM,MAAM,CAAC,IAAI,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QAExD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;YAC7B,OAAO,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC;aAC9F,OAAO,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAClC,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,OAAuE,EACvE,aAAwB,EACxB,iBAA2B,EAC3B,MAAiC,EACjC,OAAyB,EACzB,IAAY;QAEZ,MAAM,OAAO,GAAG,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACnG,MAAM,QAAQ,GAAG,2BAA2B,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzG,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;QAChE,MAAM,MAAM,GAAG,eAAe,CAC5B,wBAAwB,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAC,EACnE,2BAA2B,CAAC,OAAO,EAAE,MAAM,CAAC,CAC7C,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;QAC1F,yFAAyF;QACzF,uFAAuF;QACvF,sDAAsD;QACtD,MAAM,MAAM,GAAiB,EAAE,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;QAC/E,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC5E,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC5B,MAAM,IAAI,KAAK,CAAC,UAAU,CAAC;QAC7B,CAAC;QAED,MAAM,QAAQ,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,SAAS,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YACpC,MAAM,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACpC,MAAM,KAAK,CAAC;QACd,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACnC,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAW;QAClC,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,MAAM,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAAC,MAAM,CAAC;YACP,iGAAiG;QACnG,CAAC;QAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,+BAA+B,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;oBACvC,OAAO,EAAE,EAAE,YAAY,EAAE,2BAA2B,EAAE;iBACvD,CAAC,CAAC;gBACH,OAAO;oBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;oBAC/C,IAAI,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS;iBACtD,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,+BAA+B,EAAE,CAAC;gBACtF,MAAM,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;gBACtD,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,IAAI,MAAM,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAC7E,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CACjF,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC3C,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,GAAW,EAAE,KAAgB;QACpD,MAAM,UAAU,GAAG,SAAS,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC;QAE7D,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,+BAA+B,EAAE,OAAO,IAAI,CAAC,EAAE,CAAC;YAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,IAAI,EAAE;gBACpD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;oBACvC,OAAO,EAAE;wBACP,KAAK,EAAE,UAAU;wBACjB,YAAY,EAAE,2BAA2B;qBAC1C;iBACF,CAAC,CAAC;gBACH,OAAO;oBACL,MAAM,EAAE,QAAQ,CAAC,MAAM;oBACvB,UAAU,EAAE,QAAQ,CAAC,UAAU;oBAC/B,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC;oBAC/C,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,GAAG;wBAC5B,CAAC,CAAC,IAAI,UAAU,CAAC,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;wBAC9C,CAAC,CAAC,SAAS;iBACd,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,IAAI,qBAAqB,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,+BAA+B,EAAE,CAAC;gBACtF,MAAM,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;gBACtD,SAAS;YACX,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBACxD,MAAM,IAAI,KAAK,CACb,4CAA4C,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,EAAE,CACjF,CAAC;YACJ,CAAC;YACD,IACE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;mBACpB,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,EAChE,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,0DAA0D,UAAU,GAAG,CAAC,CAAC;YAC3F,CAAC;YACD,OAAO,MAAM,CAAC,KAAK,CAAC;QACtB,CAAC;QAED,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;CACF;AAED,SAAS,SAAS,CAAC,OAA8B;IAC/C,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;QAChD,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE;QAC9B,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;KAC3C,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpB,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAuC;IACjE,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC;QAChD,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,WAAW,EAAE;QAC9B,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,aAAa,EAAE,OAAO,CAAC,aAAa;QACpC,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;QAC5C,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,4BAA4B,CACtD,OAAO,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,EACxE,OAAO,CAAC,YAAY,CACrB;QACD,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC7B,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;gBAC7B,EAAE,EAAE,KAAK,CAAC,EAAE;gBACZ,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS;gBAChC,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;aAC3C,CAAC,CAAC;SACJ,CAAC;KACH,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,MAAiC;IACxD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACtC,GAAG,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,iBAAiB,EAAE;SAC3E,aAAa,CAAC,GAAG,KAAK,CAAC,EAAE,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;AAC1G,CAAC;AAED,KAAK,UAAU,MAAM,CAAC,IAAY;IAChC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type UpstreamAccessPolicy } from "./file-access-policy.js";
|
|
2
|
+
import { type IfsHttpAccessPolicy } from "../sources/ifs-open-data.js";
|
|
3
|
+
export declare class IfsOpenDataAccessPolicy implements IfsHttpAccessPolicy {
|
|
4
|
+
private readonly cloudPolicy;
|
|
5
|
+
private readonly directPolicy;
|
|
6
|
+
constructor(stateDir: string, cloudPolicy?: UpstreamAccessPolicy, directPolicy?: UpstreamAccessPolicy);
|
|
7
|
+
run<T>(url: string, operation: () => Promise<T>): Promise<T>;
|
|
8
|
+
private policyForUrl;
|
|
9
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { FileAccessPolicy, UPSTREAM_ACCESS_POLICIES, } from "./file-access-policy.js";
|
|
2
|
+
import { IFS_OPEN_DATA_MIRRORS, } from "../sources/ifs-open-data.js";
|
|
3
|
+
export class IfsOpenDataAccessPolicy {
|
|
4
|
+
cloudPolicy;
|
|
5
|
+
directPolicy;
|
|
6
|
+
constructor(stateDir, cloudPolicy, directPolicy) {
|
|
7
|
+
this.cloudPolicy = cloudPolicy
|
|
8
|
+
?? new FileAccessPolicy(stateDir, UPSTREAM_ACCESS_POLICIES.ecmwfCloud);
|
|
9
|
+
this.directPolicy = directPolicy
|
|
10
|
+
?? new FileAccessPolicy(stateDir, UPSTREAM_ACCESS_POLICIES.ecmwfDirect);
|
|
11
|
+
}
|
|
12
|
+
run(url, operation) {
|
|
13
|
+
return this.policyForUrl(url).run(operation);
|
|
14
|
+
}
|
|
15
|
+
policyForUrl(url) {
|
|
16
|
+
const directBaseUrl = IFS_OPEN_DATA_MIRRORS.find((mirror) => mirror.id === "ecmwf")?.baseUrl;
|
|
17
|
+
return directBaseUrl !== undefined && url.startsWith(directBaseUrl)
|
|
18
|
+
? this.directPolicy
|
|
19
|
+
: this.cloudPolicy;
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=ifs-open-data-access-policy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ifs-open-data-access-policy.js","sourceRoot":"","sources":["../../src/cache/ifs-open-data-access-policy.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,EAChB,wBAAwB,GAEzB,MAAM,yBAAyB,CAAC;AACjC,OAAO,EACL,qBAAqB,GAEtB,MAAM,6BAA6B,CAAC;AAErC,MAAM,OAAO,uBAAuB;IACjB,WAAW,CAAuB;IAClC,YAAY,CAAuB;IAEpD,YACE,QAAgB,EAChB,WAAkC,EAClC,YAAmC;QAEnC,IAAI,CAAC,WAAW,GAAG,WAAW;eACzB,IAAI,gBAAgB,CAAC,QAAQ,EAAE,wBAAwB,CAAC,UAAU,CAAC,CAAC;QACzE,IAAI,CAAC,YAAY,GAAG,YAAY;eAC3B,IAAI,gBAAgB,CAAC,QAAQ,EAAE,wBAAwB,CAAC,WAAW,CAAC,CAAC;IAC5E,CAAC;IAED,GAAG,CAAI,GAAW,EAAE,SAA2B;QAC7C,OAAO,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IAC/C,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,MAAM,aAAa,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,KAAK,OAAO,CAAC,EAAE,OAAO,CAAC;QAC7F,OAAO,aAAa,KAAK,SAAS,IAAI,GAAG,CAAC,UAAU,CAAC,aAAa,CAAC;YACjE,CAAC,CAAC,IAAI,CAAC,YAAY;YACnB,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { UpstreamAccessPolicy } from "./file-access-policy.js";
|
|
1
2
|
import { type IfsIndexSelector, type IfsOpenDataProduct } from "../sources/ifs-open-data.js";
|
|
2
3
|
export interface IfsSelectionRequest {
|
|
3
4
|
run: Date;
|
|
@@ -18,7 +19,8 @@ export declare class IfsOpenDataSubsetCache implements IfsSelectionSource {
|
|
|
18
19
|
private readonly rangeConcurrency;
|
|
19
20
|
private readonly inFlight;
|
|
20
21
|
private readonly indexInFlight;
|
|
21
|
-
|
|
22
|
+
private readonly accessPolicy;
|
|
23
|
+
constructor(rootDir: string, fetchFn?: typeof fetch, rangeConcurrency?: number, cloudAccessPolicy?: UpstreamAccessPolicy, directAccessPolicy?: UpstreamAccessPolicy);
|
|
22
24
|
fetchSelection(request: IfsSelectionRequest): Promise<IfsSubsetFile>;
|
|
23
25
|
private download;
|
|
24
26
|
private downloadFromMirror;
|