xtrawl 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/auth/bootstrap.d.ts +1 -1
- package/dist/auth/bootstrap.d.ts.map +1 -1
- package/dist/auth/bootstrap.js +27 -14
- package/dist/auth/bootstrap.js.map +1 -1
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +13 -12
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/parser.d.ts +2 -14
- package/dist/cli/parser.d.ts.map +1 -1
- package/dist/cli/parser.js +26 -30
- package/dist/cli/parser.js.map +1 -1
- package/dist/cli/types.d.ts +17 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +2 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/client/accounts.d.ts.map +1 -1
- package/dist/client/accounts.js +13 -6
- package/dist/client/accounts.js.map +1 -1
- package/dist/client/client.d.ts +13 -1
- package/dist/client/client.d.ts.map +1 -1
- package/dist/client/client.js +93 -38
- package/dist/client/client.js.map +1 -1
- package/dist/client/collectors.d.ts +2 -10
- package/dist/client/collectors.d.ts.map +1 -1
- package/dist/client/collectors.js +122 -57
- package/dist/client/collectors.js.map +1 -1
- package/dist/client/database.d.ts +6 -18
- package/dist/client/database.d.ts.map +1 -1
- package/dist/client/database.js +4 -21
- package/dist/client/database.js.map +1 -1
- package/dist/client/page-commit.d.ts +15 -0
- package/dist/client/page-commit.d.ts.map +1 -0
- package/dist/client/page-commit.js +19 -0
- package/dist/client/page-commit.js.map +1 -0
- package/dist/client/profiles.d.ts +1 -1
- package/dist/client/profiles.d.ts.map +1 -1
- package/dist/client/profiles.js +10 -3
- package/dist/client/profiles.js.map +1 -1
- package/dist/client/search.d.ts +2 -15
- package/dist/client/search.d.ts.map +1 -1
- package/dist/client/search.js +64 -25
- package/dist/client/search.js.map +1 -1
- package/dist/client/types.d.ts +40 -2
- package/dist/client/types.d.ts.map +1 -1
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +5 -2
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/types.d.ts +5 -2
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/validation.d.ts +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +27 -11
- package/dist/config/validation.js.map +1 -1
- package/dist/constants/accounts.d.ts +16 -0
- package/dist/constants/accounts.d.ts.map +1 -0
- package/dist/constants/accounts.js +16 -0
- package/dist/constants/accounts.js.map +1 -0
- package/dist/constants/cli.d.ts +10 -0
- package/dist/constants/cli.d.ts.map +1 -0
- package/dist/constants/cli.js +10 -0
- package/dist/constants/cli.js.map +1 -0
- package/dist/constants/collection.d.ts +9 -0
- package/dist/constants/collection.d.ts.map +1 -0
- package/dist/constants/collection.js +9 -0
- package/dist/constants/collection.js.map +1 -0
- package/dist/constants/config.d.ts +6 -0
- package/dist/constants/config.d.ts.map +1 -0
- package/dist/constants/config.js +6 -0
- package/dist/constants/config.js.map +1 -0
- package/dist/constants/errors.d.ts +9 -0
- package/dist/constants/errors.d.ts.map +1 -0
- package/dist/constants/errors.js +9 -0
- package/dist/constants/errors.js.map +1 -0
- package/dist/constants/output.d.ts +7 -0
- package/dist/constants/output.d.ts.map +1 -0
- package/dist/constants/output.js +7 -0
- package/dist/constants/output.js.map +1 -0
- package/dist/constants/pages.d.ts +5 -0
- package/dist/constants/pages.d.ts.map +1 -0
- package/dist/constants/pages.js +5 -0
- package/dist/constants/pages.js.map +1 -0
- package/dist/constants/requests.d.ts +18 -0
- package/dist/constants/requests.d.ts.map +1 -0
- package/dist/constants/requests.js +18 -0
- package/dist/constants/requests.js.map +1 -0
- package/dist/constants/runner.d.ts +8 -0
- package/dist/constants/runner.d.ts.map +1 -0
- package/dist/constants/runner.js +8 -0
- package/dist/constants/runner.js.map +1 -0
- package/dist/constants/runs.d.ts +8 -0
- package/dist/constants/runs.d.ts.map +1 -0
- package/dist/constants/runs.js +8 -0
- package/dist/constants/runs.js.map +1 -0
- package/dist/domain/accounts.d.ts +4 -2
- package/dist/domain/accounts.d.ts.map +1 -1
- package/dist/domain/collection.d.ts +27 -0
- package/dist/domain/collection.d.ts.map +1 -0
- package/dist/domain/collection.js +2 -0
- package/dist/domain/collection.js.map +1 -0
- package/dist/domain/diagnostics.d.ts +11 -0
- package/dist/domain/diagnostics.d.ts.map +1 -0
- package/dist/domain/diagnostics.js +2 -0
- package/dist/domain/diagnostics.js.map +1 -0
- package/dist/domain/errors.d.ts +2 -1
- package/dist/domain/errors.d.ts.map +1 -1
- package/dist/domain/errors.js +10 -9
- package/dist/domain/errors.js.map +1 -1
- package/dist/domain/http.d.ts +7 -1
- package/dist/domain/http.d.ts.map +1 -1
- package/dist/domain/manifest.d.ts +8 -0
- package/dist/domain/manifest.d.ts.map +1 -1
- package/dist/domain/output.d.ts +3 -0
- package/dist/domain/output.d.ts.map +1 -0
- package/dist/domain/output.js +2 -0
- package/dist/domain/output.js.map +1 -0
- package/dist/domain/pages.d.ts +20 -0
- package/dist/domain/pages.d.ts.map +1 -0
- package/dist/domain/pages.js +2 -0
- package/dist/domain/pages.js.map +1 -0
- package/dist/domain/pool.d.ts +22 -0
- package/dist/domain/pool.d.ts.map +1 -0
- package/dist/domain/pool.js +2 -0
- package/dist/domain/pool.js.map +1 -0
- package/dist/domain/records.d.ts +2 -1
- package/dist/domain/records.d.ts.map +1 -1
- package/dist/domain/requests.d.ts +15 -10
- package/dist/domain/requests.d.ts.map +1 -1
- package/dist/domain/runner.d.ts +30 -0
- package/dist/domain/runner.d.ts.map +1 -0
- package/dist/domain/runner.js +2 -0
- package/dist/domain/runner.js.map +1 -0
- package/dist/domain/runs.d.ts +12 -0
- package/dist/domain/runs.d.ts.map +1 -0
- package/dist/domain/runs.js +2 -0
- package/dist/domain/runs.js.map +1 -0
- package/dist/engine/api-engine.d.ts +7 -7
- package/dist/engine/api-engine.d.ts.map +1 -1
- package/dist/engine/api-engine.js +42 -19
- package/dist/engine/api-engine.js.map +1 -1
- package/dist/engine/extractors.d.ts +2 -8
- package/dist/engine/extractors.d.ts.map +1 -1
- package/dist/engine/extractors.js +30 -11
- package/dist/engine/extractors.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/manifest/destinations.d.ts +4 -0
- package/dist/manifest/destinations.d.ts.map +1 -0
- package/dist/manifest/destinations.js +47 -0
- package/dist/manifest/destinations.js.map +1 -0
- package/dist/manifest/model.d.ts +3 -2
- package/dist/manifest/model.d.ts.map +1 -1
- package/dist/manifest/model.js +6 -1
- package/dist/manifest/model.js.map +1 -1
- package/dist/manifest/provider.d.ts +10 -3
- package/dist/manifest/provider.d.ts.map +1 -1
- package/dist/manifest/provider.js +86 -34
- package/dist/manifest/provider.js.map +1 -1
- package/dist/manifest/scraper.d.ts +2 -6
- package/dist/manifest/scraper.d.ts.map +1 -1
- package/dist/manifest/scraper.js +46 -15
- package/dist/manifest/scraper.js.map +1 -1
- package/dist/output/csv-writer.d.ts.map +1 -1
- package/dist/output/csv-writer.js +32 -10
- package/dist/output/csv-writer.js.map +1 -1
- package/dist/output/json-writer.d.ts +1 -0
- package/dist/output/json-writer.d.ts.map +1 -1
- package/dist/output/json-writer.js +36 -11
- package/dist/output/json-writer.js.map +1 -1
- package/dist/output/lock.d.ts +2 -0
- package/dist/output/lock.d.ts.map +1 -0
- package/dist/output/lock.js +18 -0
- package/dist/output/lock.js.map +1 -0
- package/dist/output/writer.d.ts +2 -1
- package/dist/output/writer.d.ts.map +1 -1
- package/dist/output/writer.js +6 -3
- package/dist/output/writer.js.map +1 -1
- package/dist/pool/account-pool.d.ts +3 -12
- package/dist/pool/account-pool.d.ts.map +1 -1
- package/dist/pool/account-pool.js +54 -16
- package/dist/pool/account-pool.js.map +1 -1
- package/dist/pool/cooldown.d.ts +3 -7
- package/dist/pool/cooldown.d.ts.map +1 -1
- package/dist/pool/cooldown.js +10 -7
- package/dist/pool/cooldown.js.map +1 -1
- package/dist/pool/limiter.d.ts +3 -3
- package/dist/pool/limiter.d.ts.map +1 -1
- package/dist/pool/limiter.js +5 -6
- package/dist/pool/limiter.js.map +1 -1
- package/dist/query/builder.d.ts +1 -1
- package/dist/query/builder.d.ts.map +1 -1
- package/dist/query/builder.js +10 -4
- package/dist/query/builder.js.map +1 -1
- package/dist/query/collection-id.d.ts +3 -0
- package/dist/query/collection-id.d.ts.map +1 -0
- package/dist/query/collection-id.js +21 -0
- package/dist/query/collection-id.js.map +1 -0
- package/dist/query/normalize.d.ts +3 -32
- package/dist/query/normalize.d.ts.map +1 -1
- package/dist/query/normalize.js +10 -16
- package/dist/query/normalize.js.map +1 -1
- package/dist/query/targets.d.ts +2 -7
- package/dist/query/targets.d.ts.map +1 -1
- package/dist/query/targets.js.map +1 -1
- package/dist/query/types.d.ts +40 -0
- package/dist/query/types.d.ts.map +1 -0
- package/dist/query/types.js +2 -0
- package/dist/query/types.js.map +1 -0
- package/dist/runner/index.d.ts +3 -0
- package/dist/runner/index.d.ts.map +1 -1
- package/dist/runner/index.js.map +1 -1
- package/dist/runner/retry.d.ts +3 -6
- package/dist/runner/retry.d.ts.map +1 -1
- package/dist/runner/retry.js +5 -3
- package/dist/runner/retry.js.map +1 -1
- package/dist/runner/runner.d.ts +2 -14
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +11 -7
- package/dist/runner/runner.js.map +1 -1
- package/dist/runner/task-queue.d.ts +9 -11
- package/dist/runner/task-queue.d.ts.map +1 -1
- package/dist/runner/task-queue.js +79 -32
- package/dist/runner/task-queue.js.map +1 -1
- package/dist/storage/account-repository.d.ts +2 -15
- package/dist/storage/account-repository.d.ts.map +1 -1
- package/dist/storage/account-repository.js +65 -35
- package/dist/storage/account-repository.js.map +1 -1
- package/dist/storage/database.d.ts +3 -1
- package/dist/storage/database.d.ts.map +1 -1
- package/dist/storage/database.js +10 -4
- package/dist/storage/database.js.map +1 -1
- package/dist/storage/index.d.ts +3 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/progress-repository.d.ts +12 -0
- package/dist/storage/progress-repository.d.ts.map +1 -0
- package/dist/storage/progress-repository.js +76 -0
- package/dist/storage/progress-repository.js.map +1 -0
- package/dist/storage/run-repository.d.ts +4 -10
- package/dist/storage/run-repository.d.ts.map +1 -1
- package/dist/storage/run-repository.js +4 -2
- package/dist/storage/run-repository.js.map +1 -1
- package/dist/storage/schema.d.ts.map +1 -1
- package/dist/storage/schema.js +19 -0
- package/dist/storage/schema.js.map +1 -1
- package/dist/storage/types.d.ts +15 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +2 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/transport/body.d.ts +15 -0
- package/dist/transport/body.d.ts.map +1 -0
- package/dist/transport/body.js +61 -0
- package/dist/transport/body.js.map +1 -0
- package/dist/transport/dispatcher-pool.d.ts +13 -0
- package/dist/transport/dispatcher-pool.d.ts.map +1 -0
- package/dist/transport/dispatcher-pool.js +47 -0
- package/dist/transport/dispatcher-pool.js.map +1 -0
- package/dist/transport/graphql.d.ts +1 -1
- package/dist/transport/graphql.d.ts.map +1 -1
- package/dist/transport/graphql.js +36 -19
- package/dist/transport/graphql.js.map +1 -1
- package/dist/transport/session.d.ts +8 -11
- package/dist/transport/session.d.ts.map +1 -1
- package/dist/transport/session.js +71 -26
- package/dist/transport/session.js.map +1 -1
- package/dist/transport/transaction-id.d.ts +2 -7
- package/dist/transport/transaction-id.d.ts.map +1 -1
- package/dist/transport/transaction-id.js +20 -16
- package/dist/transport/transaction-id.js.map +1 -1
- package/dist/transport/types.d.ts +28 -0
- package/dist/transport/types.d.ts.map +1 -0
- package/dist/transport/types.js +2 -0
- package/dist/transport/types.js.map +1 -0
- package/dist/utils/abort.d.ts +6 -0
- package/dist/utils/abort.d.ts.map +1 -0
- package/dist/utils/abort.js +45 -0
- package/dist/utils/abort.js.map +1 -0
- package/dist/utils/redact.d.ts +5 -0
- package/dist/utils/redact.d.ts.map +1 -0
- package/dist/utils/redact.js +42 -0
- package/dist/utils/redact.js.map +1 -0
- package/docs/architecture/overview.md +6 -4
- package/docs/product/specification.md +10 -9
- package/docs/security/data-boundary.md +3 -0
- package/package.json +5 -4
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { AcceptedRecord, PageCommit, TaskProgress } from "../domain/collection.js";
|
|
2
|
+
import type { StateDatabase } from "./database.js";
|
|
3
|
+
export type { AcceptedRecord, PageCommit, ProgressState, TaskProgress } from "../domain/collection.js";
|
|
4
|
+
export declare class ProgressRepository {
|
|
5
|
+
private readonly database;
|
|
6
|
+
constructor(database: StateDatabase);
|
|
7
|
+
commitPage(commit: PageCommit): void;
|
|
8
|
+
task(collectionId: string, taskId: string): TaskProgress | undefined;
|
|
9
|
+
accepted(collectionId: string): readonly AcceptedRecord[];
|
|
10
|
+
hasRecord(collectionId: string, recordId: string): boolean;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=progress-repository.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress-repository.d.ts","sourceRoot":"","sources":["../../src/storage/progress-repository.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,UAAU,EAAiB,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvG,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,YAAY,EAAE,cAAc,EAAE,UAAU,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvG,qBAAa,kBAAkB;IACV,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,aAAa;IAEpD,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAyCpC,IAAI,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAiBpE,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,SAAS,cAAc,EAAE;IAazD,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO;CASlE"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { PROGRESS_STATE } from "../constants/collection.js";
|
|
2
|
+
import { COLLECTION_SCHEMA_VERSION } from "../query/collection-id.js";
|
|
3
|
+
export class ProgressRepository {
|
|
4
|
+
database;
|
|
5
|
+
constructor(database) {
|
|
6
|
+
this.database = database;
|
|
7
|
+
}
|
|
8
|
+
commitPage(commit) {
|
|
9
|
+
this.database.transaction(() => {
|
|
10
|
+
const now = new Date().toISOString();
|
|
11
|
+
for (const record of commit.records) {
|
|
12
|
+
this.database.run(`INSERT OR IGNORE INTO accepted_records(collection_id,record_id,task_id,payload_json,created_at)
|
|
13
|
+
VALUES(?,?,?,?,?)`, commit.collectionId, record.id, commit.taskId, JSON.stringify(record.payload), now);
|
|
14
|
+
}
|
|
15
|
+
const existing = this.task(commit.collectionId, commit.taskId);
|
|
16
|
+
const acceptedIds = [
|
|
17
|
+
...new Set([...(existing?.acceptedIds ?? []), ...commit.records.map((record) => record.id)]),
|
|
18
|
+
];
|
|
19
|
+
const storedCursor = commit.state === PROGRESS_STATE.CAPPED ? commit.inputCursor : commit.nextCursor;
|
|
20
|
+
this.database.run(`INSERT INTO collection_progress(
|
|
21
|
+
collection_id,task_id,schema_version,state,cursor,input_cursor,accepted_json,updated_at
|
|
22
|
+
) VALUES(?,?,?,?,?,?,?,?)
|
|
23
|
+
ON CONFLICT(collection_id,task_id) DO UPDATE SET
|
|
24
|
+
state=excluded.state,
|
|
25
|
+
cursor=excluded.cursor,
|
|
26
|
+
input_cursor=excluded.input_cursor,
|
|
27
|
+
accepted_json=excluded.accepted_json,
|
|
28
|
+
updated_at=excluded.updated_at`, commit.collectionId, commit.taskId, COLLECTION_SCHEMA_VERSION, commit.state, storedCursor ?? null, commit.inputCursor ?? null, JSON.stringify(acceptedIds), now);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
task(collectionId, taskId) {
|
|
32
|
+
const row = this.database.get("SELECT * FROM collection_progress WHERE collection_id=? AND task_id=?", collectionId, taskId);
|
|
33
|
+
if (!row)
|
|
34
|
+
return undefined;
|
|
35
|
+
return {
|
|
36
|
+
collectionId,
|
|
37
|
+
taskId,
|
|
38
|
+
state: String(row.state),
|
|
39
|
+
...(typeof row.cursor === "string" && row.cursor ? { cursor: row.cursor } : {}),
|
|
40
|
+
...(typeof row.input_cursor === "string" && row.input_cursor ? { inputCursor: row.input_cursor } : {}),
|
|
41
|
+
acceptedIds: parseStringArray(row.accepted_json),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
accepted(collectionId) {
|
|
45
|
+
return this.database
|
|
46
|
+
.all("SELECT record_id,task_id,payload_json FROM accepted_records WHERE collection_id=? ORDER BY created_at, record_id", collectionId)
|
|
47
|
+
.map((row) => ({
|
|
48
|
+
id: String(row.record_id),
|
|
49
|
+
taskId: String(row.task_id),
|
|
50
|
+
payload: parseJson(row.payload_json),
|
|
51
|
+
}));
|
|
52
|
+
}
|
|
53
|
+
hasRecord(collectionId, recordId) {
|
|
54
|
+
return Boolean(this.database.get("SELECT 1 FROM accepted_records WHERE collection_id=? AND record_id=?", collectionId, recordId));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
function parseStringArray(value) {
|
|
58
|
+
try {
|
|
59
|
+
const parsed = JSON.parse(String(value));
|
|
60
|
+
if (!Array.isArray(parsed))
|
|
61
|
+
return [];
|
|
62
|
+
return parsed.filter((item) => typeof item === "string");
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return [];
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function parseJson(value) {
|
|
69
|
+
try {
|
|
70
|
+
return JSON.parse(String(value));
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=progress-repository.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress-repository.js","sourceRoot":"","sources":["../../src/storage/progress-repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AAE5D,OAAO,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAKtE,MAAM,OAAO,kBAAkB;IACO;IAApC,YAAoC,QAAuB;QAAvB,aAAQ,GAAR,QAAQ,CAAe;IAAG,CAAC;IAExD,UAAU,CAAC,MAAkB;QAClC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,GAAG,EAAE;YAC7B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YACrC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpC,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf;6BACmB,EACnB,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,EACb,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAC9B,GAAG,CACJ,CAAC;YACJ,CAAC;YACD,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/D,MAAM,WAAW,GAAG;gBAClB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;aAC7F,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,KAAK,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC;YACrG,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf;;;;;;;;0CAQkC,EAClC,MAAM,CAAC,YAAY,EACnB,MAAM,CAAC,MAAM,EACb,yBAAyB,EACzB,MAAM,CAAC,KAAK,EACZ,YAAY,IAAI,IAAI,EACpB,MAAM,CAAC,WAAW,IAAI,IAAI,EAC1B,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAC3B,GAAG,CACJ,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEM,IAAI,CAAC,YAAoB,EAAE,MAAc;QAC9C,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAC3B,uEAAuE,EACvE,YAAY,EACZ,MAAM,CACP,CAAC;QACF,IAAI,CAAC,GAAG;YAAE,OAAO,SAAS,CAAC;QAC3B,OAAO;YACL,YAAY;YACZ,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAkB;YACzC,GAAG,CAAC,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/E,GAAG,CAAC,OAAO,GAAG,CAAC,YAAY,KAAK,QAAQ,IAAI,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,GAAG,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtG,WAAW,EAAE,gBAAgB,CAAC,GAAG,CAAC,aAAa,CAAC;SACjD,CAAC;IACJ,CAAC;IAEM,QAAQ,CAAC,YAAoB;QAClC,OAAO,IAAI,CAAC,QAAQ;aACjB,GAAG,CACF,kHAAkH,EAClH,YAAY,CACb;aACA,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACb,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;YACzB,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC;YAC3B,OAAO,EAAE,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC;SACrC,CAAC,CAAC,CAAC;IACR,CAAC;IAEM,SAAS,CAAC,YAAoB,EAAE,QAAgB;QACrD,OAAO,OAAO,CACZ,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,sEAAsE,EACtE,YAAY,EACZ,QAAQ,CACT,CACF,CAAC;IACJ,CAAC;CACF;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAY,CAAC;QACpD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,CAAC;QACtC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAY,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -1,18 +1,12 @@
|
|
|
1
|
+
import { RUN_STATUS } from "../constants/runs.js";
|
|
2
|
+
import type { RunRecord, RunStatus } from "../domain/runs.js";
|
|
1
3
|
import type { StateDatabase } from "./database.js";
|
|
2
|
-
export
|
|
3
|
-
readonly id: string;
|
|
4
|
-
readonly operation: string;
|
|
5
|
-
readonly queryHash?: string;
|
|
6
|
-
readonly startedAt: string;
|
|
7
|
-
readonly finishedAt?: string;
|
|
8
|
-
readonly status: "running" | "complete" | "failed";
|
|
9
|
-
readonly error?: unknown;
|
|
10
|
-
}
|
|
4
|
+
export type { RunRecord, RunStatus } from "../domain/runs.js";
|
|
11
5
|
export declare class RunRepository {
|
|
12
6
|
private readonly database;
|
|
13
7
|
constructor(database: StateDatabase);
|
|
14
8
|
create(operation: string, queryHash?: string): RunRecord;
|
|
15
|
-
finalize(id: string, status:
|
|
9
|
+
finalize(id: string, status: Exclude<RunStatus, typeof RUN_STATUS.RUNNING>, error?: unknown): boolean;
|
|
16
10
|
list(limit?: number): RunRecord[];
|
|
17
11
|
last(): RunRecord | undefined;
|
|
18
12
|
summary(limit?: number): {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-repository.d.ts","sourceRoot":"","sources":["../../src/storage/run-repository.ts"],"names":[],"mappings":"AACA,OAAO,
|
|
1
|
+
{"version":3,"file":"run-repository.d.ts","sourceRoot":"","sources":["../../src/storage/run-repository.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,mBAAmB,CAAC;AAE9D,qBAAa,aAAa;IACL,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,aAAa;IAEpD,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS;IAmBxD,QAAQ,CACb,EAAE,EAAE,MAAM,EACV,MAAM,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,UAAU,CAAC,OAAO,CAAC,EACrD,KAAK,CAAC,EAAE,OAAO,GACd,OAAO;IAYH,IAAI,CAAC,KAAK,SAAK,GAAG,SAAS,EAAE;IAY7B,IAAI,IAAI,SAAS,GAAG,SAAS;IAI7B,OAAO,CAAC,KAAK,SAAM,GAAG;QAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QACpD,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,CAAC;KAC9B;CAMF"}
|
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
+
import { RUN_STATUS } from "../constants/runs.js";
|
|
3
|
+
import { redactUnknown } from "../utils/redact.js";
|
|
2
4
|
export class RunRepository {
|
|
3
5
|
database;
|
|
4
6
|
constructor(database) {
|
|
@@ -10,13 +12,13 @@ export class RunRepository {
|
|
|
10
12
|
operation,
|
|
11
13
|
...(queryHash ? { queryHash } : {}),
|
|
12
14
|
startedAt: new Date().toISOString(),
|
|
13
|
-
status:
|
|
15
|
+
status: RUN_STATUS.RUNNING,
|
|
14
16
|
};
|
|
15
17
|
this.database.run("INSERT INTO runs(id,operation,query_hash,started_at,status) VALUES(?,?,?,?,?)", record.id, record.operation, record.queryHash ?? null, record.startedAt, record.status);
|
|
16
18
|
return record;
|
|
17
19
|
}
|
|
18
20
|
finalize(id, status, error) {
|
|
19
|
-
return (this.database.run("UPDATE runs SET finished_at=?, status=?, error_json=? WHERE id=?", new Date().toISOString(), status, error ? JSON.stringify(error) : null, id).changes === 1);
|
|
21
|
+
return (this.database.run("UPDATE runs SET finished_at=?, status=?, error_json=? WHERE id=?", new Date().toISOString(), status, error ? JSON.stringify(redactUnknown(error)) : null, id).changes === 1);
|
|
20
22
|
}
|
|
21
23
|
list(limit = 50) {
|
|
22
24
|
return this.database.all("SELECT * FROM runs ORDER BY started_at DESC LIMIT ?", limit).map((row) => ({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"run-repository.js","sourceRoot":"","sources":["../../src/storage/run-repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"run-repository.js","sourceRoot":"","sources":["../../src/storage/run-repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAKnD,MAAM,OAAO,aAAa;IACY;IAApC,YAAoC,QAAuB;QAAvB,aAAQ,GAAR,QAAQ,CAAe;IAAG,CAAC;IAExD,MAAM,CAAC,SAAiB,EAAE,SAAkB;QACjD,MAAM,MAAM,GAAc;YACxB,EAAE,EAAE,UAAU,EAAE;YAChB,SAAS;YACT,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,MAAM,EAAE,UAAU,CAAC,OAAO;SAC3B,CAAC;QACF,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,+EAA+E,EAC/E,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,SAAS,IAAI,IAAI,EACxB,MAAM,CAAC,SAAS,EAChB,MAAM,CAAC,MAAM,CACd,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAEM,QAAQ,CACb,EAAU,EACV,MAAqD,EACrD,KAAe;QAEf,OAAO,CACL,IAAI,CAAC,QAAQ,CAAC,GAAG,CACf,kEAAkE,EAClE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EACxB,MAAM,EACN,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EACnD,EAAE,CACH,CAAC,OAAO,KAAK,CAAC,CAChB,CAAC;IACJ,CAAC;IAEM,IAAI,CAAC,KAAK,GAAG,EAAE;QACpB,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,qDAAqD,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACnG,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAClB,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC;YAChC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAChE,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;YACjC,GAAG,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnE,MAAM,EAAE,GAAG,CAAC,MAA6B;YACzC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAChE,CAAC,CAAC,CAAC;IACN,CAAC;IAEM,IAAI;QACT,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzB,CAAC;IAEM,OAAO,CAAC,KAAK,GAAG,GAAG;QAKxB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,MAAM,QAAQ,GAA2B,EAAE,CAAC;QAC5C,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/E,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;IACxF,CAAC;CACF;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAY,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/storage/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/storage/schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAEnD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAsE9D"}
|
package/dist/storage/schema.js
CHANGED
|
@@ -48,6 +48,25 @@ export function initializeSchema(database) {
|
|
|
48
48
|
fetched_at REAL NOT NULL,
|
|
49
49
|
expires_at REAL NOT NULL
|
|
50
50
|
);
|
|
51
|
+
CREATE TABLE IF NOT EXISTS collection_progress (
|
|
52
|
+
collection_id TEXT NOT NULL,
|
|
53
|
+
task_id TEXT NOT NULL,
|
|
54
|
+
schema_version INTEGER NOT NULL,
|
|
55
|
+
state TEXT NOT NULL,
|
|
56
|
+
cursor TEXT,
|
|
57
|
+
input_cursor TEXT,
|
|
58
|
+
accepted_json TEXT NOT NULL DEFAULT '[]',
|
|
59
|
+
updated_at TEXT NOT NULL,
|
|
60
|
+
PRIMARY KEY (collection_id, task_id)
|
|
61
|
+
);
|
|
62
|
+
CREATE TABLE IF NOT EXISTS accepted_records (
|
|
63
|
+
collection_id TEXT NOT NULL,
|
|
64
|
+
record_id TEXT NOT NULL,
|
|
65
|
+
task_id TEXT NOT NULL,
|
|
66
|
+
payload_json TEXT NOT NULL,
|
|
67
|
+
created_at TEXT NOT NULL,
|
|
68
|
+
PRIMARY KEY (collection_id, record_id)
|
|
69
|
+
);
|
|
51
70
|
`);
|
|
52
71
|
}
|
|
53
72
|
//# sourceMappingURL=schema.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/storage/schema.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAAC,QAAuB;IACtD,QAAQ,CAAC,IAAI,CAAC
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/storage/schema.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,gBAAgB,CAAC,QAAuB;IACtD,QAAQ,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoEb,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { AccountStatus } from "../domain/accounts.js";
|
|
2
|
+
export type SqlRow = Record<string, unknown>;
|
|
3
|
+
export interface ReleaseOptions {
|
|
4
|
+
readonly status?: AccountStatus;
|
|
5
|
+
readonly availableUntil?: number;
|
|
6
|
+
readonly lastErrorCode?: number;
|
|
7
|
+
readonly cooldownReason?: string;
|
|
8
|
+
readonly dailyRequests?: number;
|
|
9
|
+
}
|
|
10
|
+
export interface AccountRepositoryOptions {
|
|
11
|
+
readonly dailyRequestsLimit?: number;
|
|
12
|
+
readonly dailyTweetsLimit?: number;
|
|
13
|
+
readonly leaseTtlMs?: number;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAE7C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,aAAa,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,MAAM,CAAC;IACrC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAC9B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import type { ReadableHttpResponse } from "./types.js";
|
|
2
|
+
export type { ReadableHttpResponse } from "./types.js";
|
|
3
|
+
export declare const DEFAULT_MAX_RESPONSE_BYTES: number;
|
|
4
|
+
export declare const AUXILIARY_REQUEST_TIMEOUT_MS = 15000;
|
|
5
|
+
export declare function readResponseText(response: ReadableHttpResponse, options: {
|
|
6
|
+
readonly signal?: AbortSignal;
|
|
7
|
+
readonly maxBytes: number;
|
|
8
|
+
}): Promise<string>;
|
|
9
|
+
export declare function cancelBody(response: {
|
|
10
|
+
readonly body?: {
|
|
11
|
+
cancel?(reason?: unknown): Promise<void>;
|
|
12
|
+
} | null;
|
|
13
|
+
}): Promise<void>;
|
|
14
|
+
export declare function oversizedError(): Error;
|
|
15
|
+
//# sourceMappingURL=body.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"body.d.ts","sourceRoot":"","sources":["../../src/transport/body.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEvD,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAEvD,eAAO,MAAM,0BAA0B,QAAkB,CAAC;AAC1D,eAAO,MAAM,4BAA4B,QAAS,CAAC;AAEnD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,oBAAoB,EAC9B,OAAO,EAAE;IAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACpE,OAAO,CAAC,MAAM,CAAC,CA0CjB;AAED,wBAAsB,UAAU,CAAC,QAAQ,EAAE;IACzC,QAAQ,CAAC,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;KAAE,GAAG,IAAI,CAAC;CACrE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEhB;AAED,wBAAgB,cAAc,IAAI,KAAK,CAEtC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { abortError } from "../utils/abort.js";
|
|
2
|
+
export const DEFAULT_MAX_RESPONSE_BYTES = 8 * 1024 * 1024;
|
|
3
|
+
export const AUXILIARY_REQUEST_TIMEOUT_MS = 15_000;
|
|
4
|
+
export async function readResponseText(response, options) {
|
|
5
|
+
if (options.signal?.aborted) {
|
|
6
|
+
await cancelBody(response);
|
|
7
|
+
throw abortError();
|
|
8
|
+
}
|
|
9
|
+
if (!response.body) {
|
|
10
|
+
const text = await response.text();
|
|
11
|
+
if (Buffer.byteLength(text) > options.maxBytes)
|
|
12
|
+
throw oversizedError();
|
|
13
|
+
return text;
|
|
14
|
+
}
|
|
15
|
+
const reader = response.body.getReader();
|
|
16
|
+
const chunks = [];
|
|
17
|
+
let size = 0;
|
|
18
|
+
const onAbort = () => {
|
|
19
|
+
void reader.cancel("aborted").catch(() => undefined);
|
|
20
|
+
};
|
|
21
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
22
|
+
try {
|
|
23
|
+
while (true) {
|
|
24
|
+
if (options.signal?.aborted)
|
|
25
|
+
throw abortError();
|
|
26
|
+
let chunk;
|
|
27
|
+
try {
|
|
28
|
+
chunk = await reader.read();
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
if (options.signal?.aborted)
|
|
32
|
+
throw abortError();
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
if (options.signal?.aborted)
|
|
36
|
+
throw abortError();
|
|
37
|
+
if (chunk.done)
|
|
38
|
+
break;
|
|
39
|
+
const value = chunk.value;
|
|
40
|
+
if (!value)
|
|
41
|
+
continue;
|
|
42
|
+
size += value.byteLength;
|
|
43
|
+
if (size > options.maxBytes) {
|
|
44
|
+
await reader.cancel("oversized").catch(() => undefined);
|
|
45
|
+
throw oversizedError();
|
|
46
|
+
}
|
|
47
|
+
chunks.push(value);
|
|
48
|
+
}
|
|
49
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
export async function cancelBody(response) {
|
|
56
|
+
await response.body?.cancel?.("abandoned")?.catch(() => undefined);
|
|
57
|
+
}
|
|
58
|
+
export function oversizedError() {
|
|
59
|
+
return new Error("Response exceeded the configured size limit.");
|
|
60
|
+
}
|
|
61
|
+
//# sourceMappingURL=body.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"body.js","sourceRoot":"","sources":["../../src/transport/body.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAK/C,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAC1D,MAAM,CAAC,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAEnD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,QAA8B,EAC9B,OAAqE;IAErE,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;QAC5B,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC3B,MAAM,UAAU,EAAE,CAAC;IACrB,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,QAAQ;YAAE,MAAM,cAAc,EAAE,CAAC;QACvE,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;IACzC,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,OAAO,GAAG,GAAS,EAAE;QACzB,KAAK,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IACvD,CAAC,CAAC;IACF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,UAAU,EAAE,CAAC;YAChD,IAAI,KAA2C,CAAC;YAChD,IAAI,CAAC;gBACH,KAAK,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAM,UAAU,EAAE,CAAC;gBAChD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,MAAM,UAAU,EAAE,CAAC;YAChD,IAAI,KAAK,CAAC,IAAI;gBAAE,MAAM;YACtB,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;YAC1B,IAAI,CAAC,KAAK;gBAAE,SAAS;YACrB,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;YACzB,IAAI,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC5B,MAAM,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACxD,MAAM,cAAc,EAAE,CAAC;YACzB,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChD,CAAC;YAAS,CAAC;QACT,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,QAEhC;IACC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;AACnE,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Dispatcher } from "undici";
|
|
2
|
+
import type { ProxySettings } from "../domain/accounts.js";
|
|
3
|
+
export declare class DispatcherPool {
|
|
4
|
+
private readonly entries;
|
|
5
|
+
acquire(proxy: string | ProxySettings | undefined): {
|
|
6
|
+
readonly dispatcher?: Dispatcher;
|
|
7
|
+
release(): void;
|
|
8
|
+
};
|
|
9
|
+
size(): number;
|
|
10
|
+
sweep(maxIdleMs?: number, now?: number): Promise<void>;
|
|
11
|
+
close(): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
//# sourceMappingURL=dispatcher-pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dispatcher-pool.d.ts","sourceRoot":"","sources":["../../src/transport/dispatcher-pool.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACzC,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAS3D,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAgC;IAEjD,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,GAAG;QACzD,QAAQ,CAAC,UAAU,CAAC,EAAE,UAAU,CAAC;QACjC,OAAO,IAAI,IAAI,CAAC;KACjB;IAuBM,IAAI,IAAI,MAAM;IAIR,KAAK,CAAC,SAAS,SAAS,EAAE,GAAG,SAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ1D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAIpC"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { proxyDispatcher, proxyToUrl } from "./proxy.js";
|
|
2
|
+
export class DispatcherPool {
|
|
3
|
+
entries = new Map();
|
|
4
|
+
acquire(proxy) {
|
|
5
|
+
const url = proxyToUrl(proxy);
|
|
6
|
+
if (!url)
|
|
7
|
+
return { release() { } };
|
|
8
|
+
let entry = this.entries.get(url);
|
|
9
|
+
if (!entry) {
|
|
10
|
+
const dispatcher = proxyDispatcher(url);
|
|
11
|
+
if (!dispatcher)
|
|
12
|
+
return { release() { } };
|
|
13
|
+
entry = { dispatcher, refs: 0 };
|
|
14
|
+
this.entries.set(url, entry);
|
|
15
|
+
}
|
|
16
|
+
entry.refs += 1;
|
|
17
|
+
entry.idleAt = undefined;
|
|
18
|
+
return {
|
|
19
|
+
dispatcher: entry.dispatcher,
|
|
20
|
+
release: () => {
|
|
21
|
+
const current = this.entries.get(url);
|
|
22
|
+
if (!current)
|
|
23
|
+
return;
|
|
24
|
+
current.refs -= 1;
|
|
25
|
+
if (current.refs <= 0)
|
|
26
|
+
current.idleAt = Date.now();
|
|
27
|
+
},
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
size() {
|
|
31
|
+
return this.entries.size;
|
|
32
|
+
}
|
|
33
|
+
async sweep(maxIdleMs = 60_000, now = Date.now()) {
|
|
34
|
+
for (const [key, entry] of this.entries) {
|
|
35
|
+
if (entry.refs > 0 || entry.idleAt === undefined || now - entry.idleAt < maxIdleMs)
|
|
36
|
+
continue;
|
|
37
|
+
await entry.dispatcher.close();
|
|
38
|
+
this.entries.delete(key);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
async close() {
|
|
42
|
+
for (const entry of this.entries.values())
|
|
43
|
+
await entry.dispatcher.close();
|
|
44
|
+
this.entries.clear();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
//# sourceMappingURL=dispatcher-pool.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dispatcher-pool.js","sourceRoot":"","sources":["../../src/transport/dispatcher-pool.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAQzD,MAAM,OAAO,cAAc;IACR,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;IAEjD,OAAO,CAAC,KAAyC;QAItD,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QAC9B,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,OAAO,KAAI,CAAC,EAAE,CAAC;QAClC,IAAI,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,UAAU,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC;YACxC,IAAI,CAAC,UAAU;gBAAE,OAAO,EAAE,OAAO,KAAI,CAAC,EAAE,CAAC;YACzC,KAAK,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;YAChC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC;QAChB,KAAK,CAAC,MAAM,GAAG,SAAS,CAAC;QACzB,OAAO;YACL,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,OAAO,EAAE,GAAG,EAAE;gBACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACtC,IAAI,CAAC,OAAO;oBAAE,OAAO;gBACrB,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;gBAClB,IAAI,OAAO,CAAC,IAAI,IAAI,CAAC;oBAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACrD,CAAC;SACF,CAAC;IACJ,CAAC;IAEM,IAAI;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC3B,CAAC;IAEM,KAAK,CAAC,KAAK,CAAC,SAAS,GAAG,MAAM,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE;QACrD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,GAAG,KAAK,CAAC,MAAM,GAAG,SAAS;gBAAE,SAAS;YAC7F,MAAM,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,KAAK;QAChB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE;YAAE,MAAM,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QAC1E,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -3,6 +3,6 @@ import type { TransactionIdProvider } from "./transaction-id.js";
|
|
|
3
3
|
export declare class GraphqlTransport {
|
|
4
4
|
private readonly transactions;
|
|
5
5
|
constructor(transactions: TransactionIdProvider);
|
|
6
|
-
get(session: HttpSession, url: string, params: Readonly<Record<string, string>>, timeoutMs: number): Promise<GraphqlResponse>;
|
|
6
|
+
get(session: HttpSession, url: string, params: Readonly<Record<string, string>>, timeoutMs: number, signal?: AbortSignal): Promise<GraphqlResponse>;
|
|
7
7
|
}
|
|
8
8
|
//# sourceMappingURL=graphql.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"graphql.d.ts","sourceRoot":"","sources":["../../src/transport/graphql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"graphql.d.ts","sourceRoot":"","sources":["../../src/transport/graphql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAItE,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAGjE,qBAAa,gBAAgB;IACR,OAAO,CAAC,QAAQ,CAAC,YAAY;gBAAZ,YAAY,EAAE,qBAAqB;IAE1D,GAAG,CACd,OAAO,EAAE,WAAW,EACpB,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,EACxC,SAAS,EAAE,MAAM,EACjB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,eAAe,CAAC;CA2E5B"}
|
|
@@ -1,29 +1,34 @@
|
|
|
1
1
|
import { AuthError, NetworkError, RateLimitError } from "../domain/errors.js";
|
|
2
2
|
import { isRecord } from "../utils/guards.js";
|
|
3
|
-
import {
|
|
3
|
+
import { isAbortError } from "../utils/abort.js";
|
|
4
|
+
import { isQuotaExhausted, parseRateLimitRemaining, parseRateLimitReset } from "../pool/cooldown.js";
|
|
4
5
|
export class GraphqlTransport {
|
|
5
6
|
transactions;
|
|
6
7
|
constructor(transactions) {
|
|
7
8
|
this.transactions = transactions;
|
|
8
9
|
}
|
|
9
|
-
async get(session, url, params, timeoutMs) {
|
|
10
|
+
async get(session, url, params, timeoutMs, signal) {
|
|
10
11
|
try {
|
|
11
12
|
const method = session.post ? "POST" : "GET";
|
|
12
13
|
const transactionId = await this.transactions.get(method, url);
|
|
14
|
+
const request = {
|
|
15
|
+
timeoutMs,
|
|
16
|
+
...(signal ? { signal } : {}),
|
|
17
|
+
...(transactionId ? { headers: { "X-Client-Transaction-Id": transactionId } } : {}),
|
|
18
|
+
};
|
|
13
19
|
const response = await (session.post
|
|
14
20
|
? session.post(url, {
|
|
15
21
|
body: graphqlBody(params),
|
|
16
|
-
|
|
17
|
-
...(transactionId ? { headers: { "X-Client-Transaction-Id": transactionId } } : {}),
|
|
22
|
+
...request,
|
|
18
23
|
})
|
|
19
24
|
: session.get(url, {
|
|
20
25
|
query: params,
|
|
21
|
-
|
|
22
|
-
...(transactionId ? { headers: { "X-Client-Transaction-Id": transactionId } } : {}),
|
|
26
|
+
...request,
|
|
23
27
|
}));
|
|
24
28
|
const body = await response.text();
|
|
25
|
-
const responseStatus = effectiveStatus(response.status, response.headers);
|
|
26
29
|
const snippet = body.slice(0, 240);
|
|
30
|
+
const remaining = parseRateLimitRemaining(response.headers);
|
|
31
|
+
const resetAt = parseRateLimitReset(response.headers);
|
|
27
32
|
let data = null;
|
|
28
33
|
if (response.status === 200) {
|
|
29
34
|
try {
|
|
@@ -33,19 +38,19 @@ export class GraphqlTransport {
|
|
|
33
38
|
data = null;
|
|
34
39
|
}
|
|
35
40
|
}
|
|
36
|
-
if (
|
|
37
|
-
throw new AuthError("Remote session was rejected.", { statusCode:
|
|
41
|
+
if (response.status === 401 || response.status === 403) {
|
|
42
|
+
throw new AuthError("Remote session was rejected.", { statusCode: response.status, endpoint: url });
|
|
38
43
|
}
|
|
39
|
-
if (
|
|
44
|
+
if (response.status === 429) {
|
|
40
45
|
throw new RateLimitError("Remote rate limit was returned.", {
|
|
41
|
-
statusCode:
|
|
46
|
+
statusCode: response.status,
|
|
42
47
|
endpoint: url,
|
|
43
|
-
resetAt
|
|
48
|
+
resetAt,
|
|
44
49
|
});
|
|
45
50
|
}
|
|
46
|
-
if (
|
|
47
|
-
throw new NetworkError(`GraphQL request failed with status ${
|
|
48
|
-
statusCode:
|
|
51
|
+
if (response.status >= 400) {
|
|
52
|
+
throw new NetworkError(`GraphQL request failed with status ${response.status}.`, {
|
|
53
|
+
statusCode: response.status,
|
|
49
54
|
endpoint: url,
|
|
50
55
|
});
|
|
51
56
|
}
|
|
@@ -53,14 +58,26 @@ export class GraphqlTransport {
|
|
|
53
58
|
if (mapped === 401 || mapped === 403)
|
|
54
59
|
throw new AuthError("Remote session was rejected.", { statusCode: mapped, endpoint: url });
|
|
55
60
|
if (mapped === 429)
|
|
56
|
-
throw new RateLimitError("Remote rate limit was returned.", {
|
|
57
|
-
|
|
61
|
+
throw new RateLimitError("Remote rate limit was returned.", {
|
|
62
|
+
statusCode: mapped,
|
|
63
|
+
endpoint: url,
|
|
64
|
+
resetAt,
|
|
65
|
+
});
|
|
66
|
+
return {
|
|
67
|
+
data,
|
|
68
|
+
status: mapped ?? response.status,
|
|
69
|
+
headers: response.headers,
|
|
70
|
+
snippet,
|
|
71
|
+
...(remaining === undefined ? {} : { remaining }),
|
|
72
|
+
...(resetAt === undefined ? {} : { resetAt }),
|
|
73
|
+
...(isQuotaExhausted(response.headers) ? { quotaExhausted: true } : {}),
|
|
74
|
+
};
|
|
58
75
|
}
|
|
59
76
|
catch (error) {
|
|
60
77
|
if (error instanceof AuthError || error instanceof RateLimitError)
|
|
61
78
|
throw error;
|
|
62
|
-
if (error
|
|
63
|
-
throw new NetworkError("GraphQL request timed out.", { endpoint: url });
|
|
79
|
+
if (isAbortError(error))
|
|
80
|
+
throw new NetworkError(signal?.aborted ? "GraphQL request was cancelled." : "GraphQL request timed out.", { endpoint: url, statusCode: signal?.aborted ? 499 : 408 });
|
|
64
81
|
if (error instanceof NetworkError)
|
|
65
82
|
throw error;
|
|
66
83
|
throw new NetworkError(error instanceof Error ? error.message : String(error), { endpoint: url });
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"graphql.js","sourceRoot":"","sources":["../../src/transport/graphql.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;
|
|
1
|
+
{"version":3,"file":"graphql.js","sourceRoot":"","sources":["../../src/transport/graphql.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC9E,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAEjD,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAErG,MAAM,OAAO,gBAAgB;IACS;IAApC,YAAoC,YAAmC;QAAnC,iBAAY,GAAZ,YAAY,CAAuB;IAAG,CAAC;IAEpE,KAAK,CAAC,GAAG,CACd,OAAoB,EACpB,GAAW,EACX,MAAwC,EACxC,SAAiB,EACjB,MAAoB;QAEpB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;YAC7C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC/D,MAAM,OAAO,GAAG;gBACd,SAAS;gBACT,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,yBAAyB,EAAE,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI;gBAClC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;oBAChB,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC;oBACzB,GAAG,OAAO;iBACX,CAAC;gBACJ,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;oBACf,KAAK,EAAE,MAAM;oBACb,GAAG,OAAO;iBACX,CAAC,CAAC,CAAC;YACR,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACnC,MAAM,SAAS,GAAG,uBAAuB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,mBAAmB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,IAAI,GAAY,IAAI,CAAC;YACzB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,IAAI,CAAC;oBACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,GAAG,IAAI,CAAC;gBACd,CAAC;YACH,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACvD,MAAM,IAAI,SAAS,CAAC,8BAA8B,EAAE,EAAE,UAAU,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YACtG,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBAC5B,MAAM,IAAI,cAAc,CAAC,iCAAiC,EAAE;oBAC1D,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,QAAQ,EAAE,GAAG;oBACb,OAAO;iBACR,CAAC,CAAC;YACL,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;gBAC3B,MAAM,IAAI,YAAY,CAAC,sCAAsC,QAAQ,CAAC,MAAM,GAAG,EAAE;oBAC/E,UAAU,EAAE,QAAQ,CAAC,MAAM;oBAC3B,QAAQ,EAAE,GAAG;iBACd,CAAC,CAAC;YACL,CAAC;YACD,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG;gBAClC,MAAM,IAAI,SAAS,CAAC,8BAA8B,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YAC7F,IAAI,MAAM,KAAK,GAAG;gBAChB,MAAM,IAAI,cAAc,CAAC,iCAAiC,EAAE;oBAC1D,UAAU,EAAE,MAAM;oBAClB,QAAQ,EAAE,GAAG;oBACb,OAAO;iBACR,CAAC,CAAC;YACL,OAAO;gBACL,IAAI;gBACJ,MAAM,EAAE,MAAM,IAAI,QAAQ,CAAC,MAAM;gBACjC,OAAO,EAAE,QAAQ,CAAC,OAAO;gBACzB,OAAO;gBACP,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC;gBACjD,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;gBAC7C,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,YAAY,cAAc;gBAAE,MAAM,KAAK,CAAC;YAC/E,IAAI,YAAY,CAAC,KAAK,CAAC;gBACrB,MAAM,IAAI,YAAY,CACpB,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,4BAA4B,EACjF,EAAE,QAAQ,EAAE,GAAG,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAC3D,CAAC;YACJ,IAAI,KAAK,YAAY,YAAY;gBAAE,MAAM,KAAK,CAAC;YAC/C,MAAM,IAAI,YAAY,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;QACpG,CAAC;IACH,CAAC;CACF;AAED,SAAS,WAAW,CAAC,MAAwC;IAC3D,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC1C,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAY,CAAC,CAAC;QAC7C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC,CACH,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC;QAAE,OAAO,SAAS,CAAC;IACvE,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;QACjF,IACE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC;YAC9B,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;YACrC,IAAI,KAAK,cAAc;YACvB,IAAI,KAAK,YAAY;YAErB,OAAO,GAAG,CAAC;QACb,IACE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAChC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC;YACjC,IAAI,KAAK,cAAc;YACvB,IAAI,KAAK,sBAAsB;YAE/B,OAAO,GAAG,CAAC;QACb,IAAI,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,KAAK,WAAW;YAAE,OAAO,GAAG,CAAC;IACxE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -1,18 +1,12 @@
|
|
|
1
|
-
import { fetch as undiciFetch } from "undici";
|
|
2
1
|
import type { AccountRecord, AuthMaterial, CookieMap, ProxySettings } from "../domain/accounts.js";
|
|
3
|
-
import type { HttpSession
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
readonly defaultProxy?: string | ProxySettings;
|
|
7
|
-
readonly userAgent?: string;
|
|
8
|
-
readonly httpMode?: "auto" | "async" | "sync";
|
|
9
|
-
readonly impersonate?: string;
|
|
10
|
-
readonly fetcher?: typeof undiciFetch;
|
|
11
|
-
readonly factory?: SessionFactory;
|
|
12
|
-
}
|
|
2
|
+
import type { HttpSession } from "../domain/http.js";
|
|
3
|
+
import type { SessionBuilderOptions } from "./types.js";
|
|
4
|
+
export type { SessionBuilderOptions } from "./types.js";
|
|
13
5
|
export declare class SessionBuilder {
|
|
14
6
|
private readonly options;
|
|
15
7
|
private readonly healthyProxies;
|
|
8
|
+
private readonly preflights;
|
|
9
|
+
private readonly dispatchers;
|
|
16
10
|
constructor(options: SessionBuilderOptions);
|
|
17
11
|
forAccount(account: AccountRecord): HttpSession;
|
|
18
12
|
fromMaterial(material: AuthMaterial, proxy?: string | ProxySettings): HttpSession;
|
|
@@ -20,6 +14,9 @@ export declare class SessionBuilder {
|
|
|
20
14
|
readonly url: string;
|
|
21
15
|
readonly timeoutMs: number;
|
|
22
16
|
}): Promise<void>;
|
|
17
|
+
close(): Promise<void>;
|
|
18
|
+
dispatcherCount(): number;
|
|
19
|
+
private runProxyPreflight;
|
|
23
20
|
}
|
|
24
21
|
export declare function cookieHeader(cookies: CookieMap): string;
|
|
25
22
|
//# sourceMappingURL=session.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/transport/session.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"session.d.ts","sourceRoot":"","sources":["../../src/transport/session.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACnG,OAAO,KAAK,EAAoC,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQvF,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAExD,YAAY,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAKxD,qBAAa,cAAc;IAKN,OAAO,CAAC,QAAQ,CAAC,OAAO;IAJ3C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA6B;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoC;IAC/D,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAwB;gBAEhB,OAAO,EAAE,qBAAqB;IAE3D,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,WAAW;IAK/C,YAAY,CAAC,QAAQ,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,GAAG,WAAW;IAkB3E,kBAAkB,CAC7B,OAAO,EAAE,aAAa,EACtB,OAAO,EAAE;QAAE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAC5D,OAAO,CAAC,IAAI,CAAC;IAaH,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAI5B,eAAe,IAAI,MAAM;YAIlB,iBAAiB;CAoChC;AAiGD,wBAAgB,YAAY,CAAC,OAAO,EAAE,SAAS,GAAG,MAAM,CAIvD"}
|