xtrawl 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DOCUMENTATION.md +51 -0
- package/README.md +34 -6
- package/dist/auth/bootstrap.d.ts +1 -1
- package/dist/auth/bootstrap.d.ts.map +1 -1
- package/dist/auth/bootstrap.js +27 -14
- package/dist/auth/bootstrap.js.map +1 -1
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +13 -12
- package/dist/cli/main.js.map +1 -1
- package/dist/cli/parser.d.ts +2 -14
- package/dist/cli/parser.d.ts.map +1 -1
- package/dist/cli/parser.js +26 -30
- package/dist/cli/parser.js.map +1 -1
- package/dist/cli/types.d.ts +17 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +2 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/client/accounts.d.ts.map +1 -1
- package/dist/client/accounts.js +13 -6
- package/dist/client/accounts.js.map +1 -1
- package/dist/client/client.d.ts +16 -3
- package/dist/client/client.d.ts.map +1 -1
- package/dist/client/client.js +96 -38
- package/dist/client/client.js.map +1 -1
- package/dist/client/collectors.d.ts +2 -10
- package/dist/client/collectors.d.ts.map +1 -1
- package/dist/client/collectors.js +122 -57
- package/dist/client/collectors.js.map +1 -1
- package/dist/client/database.d.ts +6 -18
- package/dist/client/database.d.ts.map +1 -1
- package/dist/client/database.js +4 -21
- package/dist/client/database.js.map +1 -1
- package/dist/client/page-commit.d.ts +15 -0
- package/dist/client/page-commit.d.ts.map +1 -0
- package/dist/client/page-commit.js +19 -0
- package/dist/client/page-commit.js.map +1 -0
- package/dist/client/profiles.d.ts +1 -1
- package/dist/client/profiles.d.ts.map +1 -1
- package/dist/client/profiles.js +10 -3
- package/dist/client/profiles.js.map +1 -1
- package/dist/client/search.d.ts +5 -17
- package/dist/client/search.d.ts.map +1 -1
- package/dist/client/search.js +81 -26
- package/dist/client/search.js.map +1 -1
- package/dist/client/types.d.ts +40 -2
- package/dist/client/types.d.ts.map +1 -1
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +5 -2
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/types.d.ts +5 -2
- package/dist/config/types.d.ts.map +1 -1
- package/dist/config/validation.d.ts +1 -1
- package/dist/config/validation.d.ts.map +1 -1
- package/dist/config/validation.js +27 -11
- package/dist/config/validation.js.map +1 -1
- package/dist/constants/accounts.d.ts +16 -0
- package/dist/constants/accounts.d.ts.map +1 -0
- package/dist/constants/accounts.js +16 -0
- package/dist/constants/accounts.js.map +1 -0
- package/dist/constants/cli.d.ts +10 -0
- package/dist/constants/cli.d.ts.map +1 -0
- package/dist/constants/cli.js +10 -0
- package/dist/constants/cli.js.map +1 -0
- package/dist/constants/collection.d.ts +9 -0
- package/dist/constants/collection.d.ts.map +1 -0
- package/dist/constants/collection.js +9 -0
- package/dist/constants/collection.js.map +1 -0
- package/dist/constants/config.d.ts +6 -0
- package/dist/constants/config.d.ts.map +1 -0
- package/dist/constants/config.js +6 -0
- package/dist/constants/config.js.map +1 -0
- package/dist/constants/errors.d.ts +9 -0
- package/dist/constants/errors.d.ts.map +1 -0
- package/dist/constants/errors.js +9 -0
- package/dist/constants/errors.js.map +1 -0
- package/dist/constants/output.d.ts +7 -0
- package/dist/constants/output.d.ts.map +1 -0
- package/dist/constants/output.js +7 -0
- package/dist/constants/output.js.map +1 -0
- package/dist/constants/pages.d.ts +5 -0
- package/dist/constants/pages.d.ts.map +1 -0
- package/dist/constants/pages.js +5 -0
- package/dist/constants/pages.js.map +1 -0
- package/dist/constants/requests.d.ts +18 -0
- package/dist/constants/requests.d.ts.map +1 -0
- package/dist/constants/requests.js +18 -0
- package/dist/constants/requests.js.map +1 -0
- package/dist/constants/runner.d.ts +8 -0
- package/dist/constants/runner.d.ts.map +1 -0
- package/dist/constants/runner.js +8 -0
- package/dist/constants/runner.js.map +1 -0
- package/dist/constants/runs.d.ts +8 -0
- package/dist/constants/runs.d.ts.map +1 -0
- package/dist/constants/runs.js +8 -0
- package/dist/constants/runs.js.map +1 -0
- package/dist/domain/accounts.d.ts +4 -2
- package/dist/domain/accounts.d.ts.map +1 -1
- package/dist/domain/collection.d.ts +27 -0
- package/dist/domain/collection.d.ts.map +1 -0
- package/dist/domain/collection.js +2 -0
- package/dist/domain/collection.js.map +1 -0
- package/dist/domain/diagnostics.d.ts +11 -0
- package/dist/domain/diagnostics.d.ts.map +1 -0
- package/dist/domain/diagnostics.js +2 -0
- package/dist/domain/diagnostics.js.map +1 -0
- package/dist/domain/errors.d.ts +2 -1
- package/dist/domain/errors.d.ts.map +1 -1
- package/dist/domain/errors.js +10 -9
- package/dist/domain/errors.js.map +1 -1
- package/dist/domain/http.d.ts +7 -1
- package/dist/domain/http.d.ts.map +1 -1
- package/dist/domain/manifest.d.ts +8 -0
- package/dist/domain/manifest.d.ts.map +1 -1
- package/dist/domain/output.d.ts +3 -0
- package/dist/domain/output.d.ts.map +1 -0
- package/dist/domain/output.js +2 -0
- package/dist/domain/output.js.map +1 -0
- package/dist/domain/pages.d.ts +20 -0
- package/dist/domain/pages.d.ts.map +1 -0
- package/dist/domain/pages.js +2 -0
- package/dist/domain/pages.js.map +1 -0
- package/dist/domain/pool.d.ts +22 -0
- package/dist/domain/pool.d.ts.map +1 -0
- package/dist/domain/pool.js +2 -0
- package/dist/domain/pool.js.map +1 -0
- package/dist/domain/records.d.ts +6 -1
- package/dist/domain/records.d.ts.map +1 -1
- package/dist/domain/requests.d.ts +19 -10
- package/dist/domain/requests.d.ts.map +1 -1
- package/dist/domain/runner.d.ts +30 -0
- package/dist/domain/runner.d.ts.map +1 -0
- package/dist/domain/runner.js +2 -0
- package/dist/domain/runner.js.map +1 -0
- package/dist/domain/runs.d.ts +12 -0
- package/dist/domain/runs.d.ts.map +1 -0
- package/dist/domain/runs.js +2 -0
- package/dist/domain/runs.js.map +1 -0
- package/dist/engine/api-engine.d.ts +7 -7
- package/dist/engine/api-engine.d.ts.map +1 -1
- package/dist/engine/api-engine.js +42 -19
- package/dist/engine/api-engine.js.map +1 -1
- package/dist/engine/extractors.d.ts +2 -8
- package/dist/engine/extractors.d.ts.map +1 -1
- package/dist/engine/extractors.js +30 -11
- package/dist/engine/extractors.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/manifest/destinations.d.ts +4 -0
- package/dist/manifest/destinations.d.ts.map +1 -0
- package/dist/manifest/destinations.js +47 -0
- package/dist/manifest/destinations.js.map +1 -0
- package/dist/manifest/model.d.ts +3 -2
- package/dist/manifest/model.d.ts.map +1 -1
- package/dist/manifest/model.js +6 -1
- package/dist/manifest/model.js.map +1 -1
- package/dist/manifest/provider.d.ts +10 -3
- package/dist/manifest/provider.d.ts.map +1 -1
- package/dist/manifest/provider.js +86 -34
- package/dist/manifest/provider.js.map +1 -1
- package/dist/manifest/scraper.d.ts +2 -6
- package/dist/manifest/scraper.d.ts.map +1 -1
- package/dist/manifest/scraper.js +46 -15
- package/dist/manifest/scraper.js.map +1 -1
- package/dist/output/csv-writer.d.ts.map +1 -1
- package/dist/output/csv-writer.js +32 -10
- package/dist/output/csv-writer.js.map +1 -1
- package/dist/output/json-writer.d.ts +1 -0
- package/dist/output/json-writer.d.ts.map +1 -1
- package/dist/output/json-writer.js +36 -11
- package/dist/output/json-writer.js.map +1 -1
- package/dist/output/lock.d.ts +2 -0
- package/dist/output/lock.d.ts.map +1 -0
- package/dist/output/lock.js +18 -0
- package/dist/output/lock.js.map +1 -0
- package/dist/output/writer.d.ts +2 -1
- package/dist/output/writer.d.ts.map +1 -1
- package/dist/output/writer.js +6 -3
- package/dist/output/writer.js.map +1 -1
- package/dist/pool/account-pool.d.ts +3 -12
- package/dist/pool/account-pool.d.ts.map +1 -1
- package/dist/pool/account-pool.js +54 -16
- package/dist/pool/account-pool.js.map +1 -1
- package/dist/pool/cooldown.d.ts +3 -7
- package/dist/pool/cooldown.d.ts.map +1 -1
- package/dist/pool/cooldown.js +10 -7
- package/dist/pool/cooldown.js.map +1 -1
- package/dist/pool/limiter.d.ts +3 -3
- package/dist/pool/limiter.d.ts.map +1 -1
- package/dist/pool/limiter.js +5 -6
- package/dist/pool/limiter.js.map +1 -1
- package/dist/query/builder.d.ts +1 -1
- package/dist/query/builder.d.ts.map +1 -1
- package/dist/query/builder.js +10 -4
- package/dist/query/builder.js.map +1 -1
- package/dist/query/collection-id.d.ts +3 -0
- package/dist/query/collection-id.d.ts.map +1 -0
- package/dist/query/collection-id.js +21 -0
- package/dist/query/collection-id.js.map +1 -0
- package/dist/query/normalize.d.ts +3 -32
- package/dist/query/normalize.d.ts.map +1 -1
- package/dist/query/normalize.js +10 -16
- package/dist/query/normalize.js.map +1 -1
- package/dist/query/targets.d.ts +2 -7
- package/dist/query/targets.d.ts.map +1 -1
- package/dist/query/targets.js.map +1 -1
- package/dist/query/types.d.ts +40 -0
- package/dist/query/types.d.ts.map +1 -0
- package/dist/query/types.js +2 -0
- package/dist/query/types.js.map +1 -0
- package/dist/runner/index.d.ts +3 -0
- package/dist/runner/index.d.ts.map +1 -1
- package/dist/runner/index.js.map +1 -1
- package/dist/runner/retry.d.ts +3 -6
- package/dist/runner/retry.d.ts.map +1 -1
- package/dist/runner/retry.js +5 -3
- package/dist/runner/retry.js.map +1 -1
- package/dist/runner/runner.d.ts +2 -14
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +11 -7
- package/dist/runner/runner.js.map +1 -1
- package/dist/runner/task-queue.d.ts +9 -11
- package/dist/runner/task-queue.d.ts.map +1 -1
- package/dist/runner/task-queue.js +79 -32
- package/dist/runner/task-queue.js.map +1 -1
- package/dist/storage/account-repository.d.ts +2 -15
- package/dist/storage/account-repository.d.ts.map +1 -1
- package/dist/storage/account-repository.js +65 -35
- package/dist/storage/account-repository.js.map +1 -1
- package/dist/storage/database.d.ts +3 -1
- package/dist/storage/database.d.ts.map +1 -1
- package/dist/storage/database.js +10 -4
- package/dist/storage/database.js.map +1 -1
- package/dist/storage/index.d.ts +3 -1
- package/dist/storage/index.d.ts.map +1 -1
- package/dist/storage/index.js +2 -0
- package/dist/storage/index.js.map +1 -1
- package/dist/storage/progress-repository.d.ts +12 -0
- package/dist/storage/progress-repository.d.ts.map +1 -0
- package/dist/storage/progress-repository.js +76 -0
- package/dist/storage/progress-repository.js.map +1 -0
- package/dist/storage/run-repository.d.ts +4 -10
- package/dist/storage/run-repository.d.ts.map +1 -1
- package/dist/storage/run-repository.js +4 -2
- package/dist/storage/run-repository.js.map +1 -1
- package/dist/storage/schema.d.ts.map +1 -1
- package/dist/storage/schema.js +19 -0
- package/dist/storage/schema.js.map +1 -1
- package/dist/storage/types.d.ts +15 -0
- package/dist/storage/types.d.ts.map +1 -0
- package/dist/storage/types.js +2 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/transport/body.d.ts +15 -0
- package/dist/transport/body.d.ts.map +1 -0
- package/dist/transport/body.js +61 -0
- package/dist/transport/body.js.map +1 -0
- package/dist/transport/dispatcher-pool.d.ts +13 -0
- package/dist/transport/dispatcher-pool.d.ts.map +1 -0
- package/dist/transport/dispatcher-pool.js +47 -0
- package/dist/transport/dispatcher-pool.js.map +1 -0
- package/dist/transport/graphql.d.ts +1 -1
- package/dist/transport/graphql.d.ts.map +1 -1
- package/dist/transport/graphql.js +36 -19
- package/dist/transport/graphql.js.map +1 -1
- package/dist/transport/session.d.ts +8 -11
- package/dist/transport/session.d.ts.map +1 -1
- package/dist/transport/session.js +71 -26
- package/dist/transport/session.js.map +1 -1
- package/dist/transport/transaction-id.d.ts +2 -7
- package/dist/transport/transaction-id.d.ts.map +1 -1
- package/dist/transport/transaction-id.js +20 -16
- package/dist/transport/transaction-id.js.map +1 -1
- package/dist/transport/types.d.ts +28 -0
- package/dist/transport/types.d.ts.map +1 -0
- package/dist/transport/types.js +2 -0
- package/dist/transport/types.js.map +1 -0
- package/dist/utils/abort.d.ts +6 -0
- package/dist/utils/abort.d.ts.map +1 -0
- package/dist/utils/abort.js +45 -0
- package/dist/utils/abort.js.map +1 -0
- package/dist/utils/redact.d.ts +5 -0
- package/dist/utils/redact.d.ts.map +1 -0
- package/dist/utils/redact.js +42 -0
- package/dist/utils/redact.js.map +1 -0
- package/docs/architecture/overview.md +8 -4
- package/docs/product/specification.md +18 -8
- package/docs/security/data-boundary.md +3 -0
- package/package.json +5 -4
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { delay } from "../utils/abort.js";
|
|
1
2
|
export class TransactionIdProvider {
|
|
2
3
|
source;
|
|
3
4
|
options;
|
|
@@ -45,7 +46,25 @@ export class TransactionIdProvider {
|
|
|
45
46
|
async initialize() {
|
|
46
47
|
try {
|
|
47
48
|
const { ClientTransaction, fetchXDocument } = await import("x-client-transaction-id");
|
|
48
|
-
|
|
49
|
+
// fetchXDocument cannot be cancelled. One bootstrap stays in-flight; a timeout
|
|
50
|
+
// abandons the wait and keeps the result if it later succeeds.
|
|
51
|
+
const work = fetchXDocument().then((document) => ClientTransaction.create(document));
|
|
52
|
+
const result = await Promise.race([
|
|
53
|
+
work.then((client) => ({ ok: true, client })),
|
|
54
|
+
delay(10_000).then(() => ({ ok: false })),
|
|
55
|
+
]);
|
|
56
|
+
if (!result.ok) {
|
|
57
|
+
void work
|
|
58
|
+
.then((client) => {
|
|
59
|
+
this.client = client;
|
|
60
|
+
this.readyAt = Date.now();
|
|
61
|
+
this.retryAt = 0;
|
|
62
|
+
})
|
|
63
|
+
.catch(() => undefined);
|
|
64
|
+
this.retryAt = Date.now() + 5 * 60_000;
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
this.client = result.client;
|
|
49
68
|
this.readyAt = Date.now();
|
|
50
69
|
this.retryAt = 0;
|
|
51
70
|
return this.client;
|
|
@@ -56,19 +75,4 @@ export class TransactionIdProvider {
|
|
|
56
75
|
}
|
|
57
76
|
}
|
|
58
77
|
}
|
|
59
|
-
async function withTimeout(value, timeoutMs) {
|
|
60
|
-
let timer;
|
|
61
|
-
try {
|
|
62
|
-
return await Promise.race([
|
|
63
|
-
value,
|
|
64
|
-
new Promise((_resolve, reject) => {
|
|
65
|
-
timer = setTimeout(() => reject(new Error("Transaction bootstrap timed out.")), timeoutMs);
|
|
66
|
-
}),
|
|
67
|
-
]);
|
|
68
|
-
}
|
|
69
|
-
finally {
|
|
70
|
-
if (timer)
|
|
71
|
-
clearTimeout(timer);
|
|
72
|
-
}
|
|
73
|
-
}
|
|
74
78
|
//# sourceMappingURL=transaction-id.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transaction-id.js","sourceRoot":"","sources":["../../src/transport/transaction-id.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"transaction-id.js","sourceRoot":"","sources":["../../src/transport/transaction-id.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAK1C,MAAM,OAAO,qBAAqB;IAOb;IACA;IAPX,MAAM,CAAqB;IAC3B,OAAO,GAAG,CAAC,CAAC;IACZ,OAAO,GAAG,CAAC,CAAC;IACZ,YAAY,CAA0C;IAE9D,YACmB,MAA4B,EAC5B,UAAgC,EAAE;QADlC,WAAM,GAAN,MAAM,CAAsB;QAC5B,YAAO,GAAP,OAAO,CAA2B;IAClD,CAAC;IAEG,KAAK,CAAC,GAAG,CAAC,MAAc,EAAE,GAAW;QAC1C,IAAI,IAAI,CAAC,MAAM,EAAE,MAAM;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAChE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO,SAAS,CAAC;QACpD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM;YAAE,OAAO,SAAS,CAAC;QAC9B,IAAI,CAAC;YACH,OAAO,MAAM,MAAM,CAAC,qBAAqB,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC3E,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,MAAM,GAAG,SAAS,CAAC;YACxB,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;YACjB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa;QACzB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC;QACxE,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,OAAO,GAAG,GAAG;YAAE,OAAO,IAAI,CAAC,MAAM,CAAC;QACvE,IAAI,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE;YAAE,OAAO,SAAS,CAAC;QAChD,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAC9D,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,YAAY,CAAC;QACjC,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,YAAY,GAAG,SAAS,CAAC;QAChC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,CAAC;YACH,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;YACtF,+EAA+E;YAC/E,+DAA+D;YAC/D,MAAM,IAAI,GAAG,cAAc,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACrF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAa,EAAE,MAAM,EAAE,CAAC,CAAC;gBACtD,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAc,EAAE,CAAC,CAAC;aACnD,CAAC,CAAC;YACH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,KAAK,IAAI;qBACN,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;oBACf,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;oBACrB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;oBAC1B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;gBACnB,CAAC,CAAC;qBACD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;gBACvC,OAAO,SAAS,CAAC;YACnB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;YACjB,OAAO,IAAI,CAAC,MAAM,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC;YACvC,OAAO,SAAS,CAAC;QACnB,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { fetch as UndiciFetch } from "undici";
|
|
2
|
+
import type { ProxySettings } from "../domain/accounts.js";
|
|
3
|
+
import type { ApiHttpMode } from "../config/types.js";
|
|
4
|
+
import type { SessionFactory } from "../domain/http.js";
|
|
5
|
+
export interface SessionBuilderOptions {
|
|
6
|
+
readonly bearerToken: string;
|
|
7
|
+
readonly defaultProxy?: string | ProxySettings;
|
|
8
|
+
readonly userAgent?: string;
|
|
9
|
+
readonly httpMode?: ApiHttpMode;
|
|
10
|
+
readonly impersonate?: string;
|
|
11
|
+
readonly fetcher?: typeof UndiciFetch;
|
|
12
|
+
readonly factory?: SessionFactory;
|
|
13
|
+
}
|
|
14
|
+
export interface ReadableHttpResponse {
|
|
15
|
+
readonly body?: {
|
|
16
|
+
getReader(): ReadableStreamDefaultReader<Uint8Array>;
|
|
17
|
+
cancel?(reason?: unknown): Promise<void>;
|
|
18
|
+
} | null;
|
|
19
|
+
text(): Promise<string>;
|
|
20
|
+
}
|
|
21
|
+
export interface TransactionIdSource {
|
|
22
|
+
readonly create?: (method: string, url: string) => Promise<string | undefined>;
|
|
23
|
+
}
|
|
24
|
+
export interface TransactionIdOptions {
|
|
25
|
+
readonly enabled?: boolean;
|
|
26
|
+
readonly ttlMs?: number;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/transport/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,IAAI,WAAW,EAAE,MAAM,QAAQ,CAAC;AACnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC;IAC/C,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,QAAQ,CAAC,EAAE,WAAW,CAAC;IAChC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,WAAW,CAAC;IACtC,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC;CACnC;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,IAAI,CAAC,EAAE;QACd,SAAS,IAAI,2BAA2B,CAAC,UAAU,CAAC,CAAC;QACrD,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;KAC1C,GAAG,IAAI,CAAC;IACT,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;CAChF;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/transport/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export declare function isAbortError(error: unknown): boolean;
|
|
2
|
+
export declare function abortError(message?: string): Error;
|
|
3
|
+
export declare function throwIfAborted(signal?: AbortSignal): void;
|
|
4
|
+
export declare function combineSignals(...signals: Array<AbortSignal | undefined>): AbortSignal | undefined;
|
|
5
|
+
export declare function delay(ms: number, signal?: AbortSignal): Promise<void>;
|
|
6
|
+
//# sourceMappingURL=abort.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"abort.d.ts","sourceRoot":"","sources":["../../src/utils/abort.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEpD;AAED,wBAAgB,UAAU,CAAC,OAAO,SAA+B,GAAG,KAAK,CAIxE;AAED,wBAAgB,cAAc,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,IAAI,CAEzD;AAED,wBAAgB,cAAc,CAAC,GAAG,OAAO,EAAE,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,GAAG,WAAW,GAAG,SAAS,CAMlG;AAED,wBAAsB,KAAK,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAoB3E"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
export function isAbortError(error) {
|
|
2
|
+
return error instanceof Error && error.name === "AbortError";
|
|
3
|
+
}
|
|
4
|
+
export function abortError(message = "The operation was aborted.") {
|
|
5
|
+
const error = new Error(message);
|
|
6
|
+
error.name = "AbortError";
|
|
7
|
+
return error;
|
|
8
|
+
}
|
|
9
|
+
export function throwIfAborted(signal) {
|
|
10
|
+
if (signal?.aborted)
|
|
11
|
+
throw abortError();
|
|
12
|
+
}
|
|
13
|
+
export function combineSignals(...signals) {
|
|
14
|
+
const active = signals.filter((signal) => Boolean(signal));
|
|
15
|
+
const first = active[0];
|
|
16
|
+
if (!first)
|
|
17
|
+
return undefined;
|
|
18
|
+
if (active.length === 1)
|
|
19
|
+
return first;
|
|
20
|
+
return AbortSignal.any(active);
|
|
21
|
+
}
|
|
22
|
+
export async function delay(ms, signal) {
|
|
23
|
+
throwIfAborted(signal);
|
|
24
|
+
if (ms <= 0)
|
|
25
|
+
return;
|
|
26
|
+
await new Promise((resolve, reject) => {
|
|
27
|
+
const timer = setTimeout(() => {
|
|
28
|
+
signal?.removeEventListener("abort", onAbort);
|
|
29
|
+
resolve();
|
|
30
|
+
}, ms);
|
|
31
|
+
const onAbort = () => {
|
|
32
|
+
clearTimeout(timer);
|
|
33
|
+
reject(abortError());
|
|
34
|
+
};
|
|
35
|
+
if (!signal)
|
|
36
|
+
return;
|
|
37
|
+
if (signal.aborted) {
|
|
38
|
+
clearTimeout(timer);
|
|
39
|
+
reject(abortError());
|
|
40
|
+
return;
|
|
41
|
+
}
|
|
42
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=abort.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"abort.js","sourceRoot":"","sources":["../../src/utils/abort.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY,CAAC,KAAc;IACzC,OAAO,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAAO,GAAG,4BAA4B;IAC/D,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,KAAK,CAAC,IAAI,GAAG,YAAY,CAAC;IAC1B,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,MAAoB;IACjD,IAAI,MAAM,EAAE,OAAO;QAAE,MAAM,UAAU,EAAE,CAAC;AAC1C,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAG,OAAuC;IACvE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAyB,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACxB,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,OAAO,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,EAAU,EAAE,MAAoB;IAC1D,cAAc,CAAC,MAAM,CAAC,CAAC;IACvB,IAAI,EAAE,IAAI,CAAC;QAAE,OAAO;IACpB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC9C,OAAO,EAAE,CAAC;QACZ,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QACvB,CAAC,CAAC;QACF,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QACD,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC5D,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { ProxySettings } from "../domain/accounts.js";
|
|
2
|
+
export declare function redactProxy(proxy: string | ProxySettings): string | ProxySettings;
|
|
3
|
+
export declare function redactText(value: string): string;
|
|
4
|
+
export declare function redactUnknown(value: unknown): unknown;
|
|
5
|
+
//# sourceMappingURL=redact.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.d.ts","sourceRoot":"","sources":["../../src/utils/redact.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAI3D,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,GAAG,MAAM,GAAG,aAAa,CAejF;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAUhD;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAUrD"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
const REDACTED = "[redacted]";
|
|
2
|
+
export function redactProxy(proxy) {
|
|
3
|
+
if (typeof proxy !== "string")
|
|
4
|
+
return {
|
|
5
|
+
...proxy,
|
|
6
|
+
...(proxy.username ? { username: REDACTED } : {}),
|
|
7
|
+
...(proxy.password ? { password: REDACTED } : {}),
|
|
8
|
+
};
|
|
9
|
+
try {
|
|
10
|
+
const value = new URL(proxy.includes("://") ? proxy : `http://${proxy}`);
|
|
11
|
+
if (value.username)
|
|
12
|
+
value.username = REDACTED;
|
|
13
|
+
if (value.password)
|
|
14
|
+
value.password = REDACTED;
|
|
15
|
+
return value.toString();
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return REDACTED;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function redactText(value) {
|
|
22
|
+
return value
|
|
23
|
+
.replace(/auth_token=[^;\s&]+/giu, `auth_token=${REDACTED}`)
|
|
24
|
+
.replace(/\bct0=[^;\s&]+/giu, `ct0=${REDACTED}`)
|
|
25
|
+
.replace(/Bearer\s+[A-Za-z0-9._%~+-]+/giu, `Bearer ${REDACTED}`)
|
|
26
|
+
.replace(/Cookie:\s*[^\n]+/giu, `Cookie: ${REDACTED}`)
|
|
27
|
+
.replace(/(?:password|passwd|secret|csrf|ct0|auth_token)\s*[:=]\s*([^\s,;]+)/giu, (match, secret) => match.replace(secret, REDACTED));
|
|
28
|
+
}
|
|
29
|
+
export function redactUnknown(value) {
|
|
30
|
+
if (typeof value === "string")
|
|
31
|
+
return redactText(value);
|
|
32
|
+
if (Array.isArray(value))
|
|
33
|
+
return value.map(redactUnknown);
|
|
34
|
+
if (!value || typeof value !== "object")
|
|
35
|
+
return value;
|
|
36
|
+
return Object.fromEntries(Object.entries(value).map(([key, child]) => {
|
|
37
|
+
if (/password|token|cookie|secret|authorization|csrf|proxy/iu.test(key))
|
|
38
|
+
return [key, REDACTED];
|
|
39
|
+
return [key, redactUnknown(child)];
|
|
40
|
+
}));
|
|
41
|
+
}
|
|
42
|
+
//# sourceMappingURL=redact.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"redact.js","sourceRoot":"","sources":["../../src/utils/redact.ts"],"names":[],"mappings":"AAEA,MAAM,QAAQ,GAAG,YAAY,CAAC;AAE9B,MAAM,UAAU,WAAW,CAAC,KAA6B;IACvD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAC3B,OAAO;YACL,GAAG,KAAK;YACR,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAC;IACJ,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,KAAK,EAAE,CAAC,CAAC;QACzE,IAAI,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC9C,IAAI,KAAK,CAAC,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC9C,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,wBAAwB,EAAE,cAAc,QAAQ,EAAE,CAAC;SAC3D,OAAO,CAAC,mBAAmB,EAAE,OAAO,QAAQ,EAAE,CAAC;SAC/C,OAAO,CAAC,gCAAgC,EAAE,UAAU,QAAQ,EAAE,CAAC;SAC/D,OAAO,CAAC,qBAAqB,EAAE,WAAW,QAAQ,EAAE,CAAC;SACrD,OAAO,CACN,uEAAuE,EACvE,CAAC,KAAK,EAAE,MAAc,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,CAC3D,CAAC;AACN,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IACxD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;IAC1D,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IACtD,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QACzC,IAAI,yDAAyD,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAChG,OAAO,CAAC,GAAG,EAAE,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
|
|
@@ -32,13 +32,16 @@ CLI / library facade
|
|
|
32
32
|
- `pool` owns per-page leases, heartbeats, cooldowns, token-bucket spacing, retries, session repair,
|
|
33
33
|
proxy preflight, and account selection.
|
|
34
34
|
- `runner` coordinates bounded concurrent targets and search intervals through interfaces.
|
|
35
|
-
- `client` composes concrete adapters and exposes the stable public API.
|
|
35
|
+
- `client` composes concrete adapters and exposes the stable public API. Its low-level search-page
|
|
36
|
+
primitive performs one pooled page request; high-level search adds scheduling, deduplication,
|
|
37
|
+
durable page progress, output, and run statistics around that same primitive. Public methods
|
|
38
|
+
accept an AbortSignal and `shutdown()` drains in-flight work.
|
|
36
39
|
- `cli` parses arguments and renders results; it never implements scraping logic.
|
|
37
40
|
|
|
38
41
|
## Persistence topology
|
|
39
42
|
|
|
40
|
-
One configured SQLite database always contains run records, resume checkpoints,
|
|
41
|
-
payloads. By default it also contains accounts and leases. A caller may replace the account-state
|
|
43
|
+
One configured SQLite database always contains run records, resume checkpoints, accepted collection
|
|
44
|
+
records, and cached manifest payloads. By default it also contains accounts and leases. A caller may replace the account-state
|
|
42
45
|
boundary with an `AccountStateStore`; the pool awaits either synchronous or asynchronous adapters and
|
|
43
46
|
requires atomic acquire, renew, completion, and replacement semantics. SQL remains local to storage.
|
|
44
47
|
Raw secrets are stored only when the caller explicitly provisions or restores them; logs and normal
|
|
@@ -51,7 +54,8 @@ projections use redaction or a token fingerprint.
|
|
|
51
54
|
3. Remote text, manifests, and headers are data, never executable instructions.
|
|
52
55
|
4. Secrets cross only the session builder and selected account-state boundary. Public inspection and
|
|
53
56
|
normal maintenance projections redact them. Explicit account-state export is the sole raw-secret
|
|
54
|
-
projection and requires caller acknowledgement.
|
|
57
|
+
projection and requires caller acknowledgement. Remote manifests cannot attach credentials to
|
|
58
|
+
unapproved origins. Authenticated requests do not follow redirects.
|
|
55
59
|
|
|
56
60
|
## Verification architecture
|
|
57
61
|
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
**Status:** approved for implementation
|
|
4
4
|
**Owner:** project user
|
|
5
|
-
**Last reviewed:** 2026-08-
|
|
5
|
+
**Last reviewed:** 2026-08-12
|
|
6
6
|
|
|
7
7
|
## Product hypothesis
|
|
8
8
|
|
|
@@ -12,11 +12,11 @@ persisting progress locally, and exposing predictable typed results.
|
|
|
12
12
|
|
|
13
13
|
## Users and outcomes
|
|
14
14
|
|
|
15
|
-
| User
|
|
16
|
-
|
|
|
17
|
-
| TypeScript developer | Existing clients are difficult to type, test, and extend
|
|
18
|
-
| Data operator
|
|
19
|
-
| Maintainer
|
|
15
|
+
| User | Problem | Successful outcome |
|
|
16
|
+
| -------------------- | ----------------------------------------------------------- | --------------------------------------------------------- |
|
|
17
|
+
| TypeScript developer | Existing clients are difficult to type, test, and extend | Import a small typed API and receive stable records |
|
|
18
|
+
| Data operator | Long reads fail when a session is cooled or a process stops | Pool accounts, retry safely, and resume from checkpoints |
|
|
19
|
+
| Maintainer | Transport, persistence, and domain behavior become coupled | Each boundary is replaceable and covered by focused tests |
|
|
20
20
|
|
|
21
21
|
## Principal journeys
|
|
22
22
|
|
|
@@ -27,6 +27,8 @@ persisting progress locally, and exposing predictable typed results.
|
|
|
27
27
|
5. Run the CLI or library with deterministic typed errors and no mutation of the remote account.
|
|
28
28
|
6. Replace SQLite account persistence with a caller-owned store and explicitly export or restore
|
|
29
29
|
reusable session state without restoring stale leases or login-only credentials.
|
|
30
|
+
7. Request one search page with an optional caller-owned cursor so an application can persist and
|
|
31
|
+
resume pagination independently of XTrawl checkpoints.
|
|
30
32
|
|
|
31
33
|
## First-release scope
|
|
32
34
|
|
|
@@ -43,6 +45,8 @@ persisting progress locally, and exposing predictable typed results.
|
|
|
43
45
|
- Strict TypeScript interfaces, runtime guards at external boundaries, and test doubles for all network paths.
|
|
44
46
|
- A pluggable sync-or-async account-state contract with atomic lease operations; SQLite remains the
|
|
45
47
|
default adapter while run history, checkpoints, and manifest cache remain local.
|
|
48
|
+
- A low-level `searchPage()` method that supports the full search-filter vocabulary and returns
|
|
49
|
+
normalized posts plus an opaque `nextCursor`; high-level `search()` remains automatic.
|
|
46
50
|
|
|
47
51
|
## Explicit exclusions
|
|
48
52
|
|
|
@@ -81,7 +85,9 @@ persisting progress locally, and exposing predictable typed results.
|
|
|
81
85
|
- [x] Harness structural validation and context-budget evaluation pass.
|
|
82
86
|
- [x] A fresh agent can identify the current work and next action from `.harness/state/` without chat history.
|
|
83
87
|
- [x] A caller-owned asynchronous account store can provision, lease, complete, export, and restore
|
|
84
|
-
|
|
88
|
+
account state through the public API, with deterministic contract tests.
|
|
89
|
+
- [x] A caller can pass a search cursor, receive the next cursor, and paginate without creating run
|
|
90
|
+
records or reading and writing the internal checkpoint store.
|
|
85
91
|
|
|
86
92
|
## Open decisions
|
|
87
93
|
|
|
@@ -89,7 +95,11 @@ persisting progress locally, and exposing predictable typed results.
|
|
|
89
95
|
- Async methods are the canonical library surface. A blocking synchronous wrapper is intentionally excluded because it would compromise Node event-loop safety.
|
|
90
96
|
- Account-state snapshots require an explicit secret acknowledgement, are runtime validated and
|
|
91
97
|
versioned, and exclude active leases plus password, email-password, and two-factor login fields.
|
|
98
|
+
- `searchPage()` exposes normalized posts and only the opaque next cursor. Query metadata and
|
|
99
|
+
persistence remain caller-owned; `search()` composes the same page primitive for automatic runs.
|
|
92
100
|
|
|
93
101
|
## Immediate next action
|
|
94
102
|
|
|
95
|
-
|
|
103
|
+
Add the read-only MCP server for agent-harness operation after the recovery, cancellation, and
|
|
104
|
+
credential-destination bounds. Keep credentials out of tool results and preserve account-pool
|
|
105
|
+
limits. MCP remains a follow-on product task.
|
|
@@ -17,6 +17,9 @@
|
|
|
17
17
|
- HTTP, HTTPS, and SOCKS5 proxies are normalized and passed to the HTTP adapter. Health checks carry
|
|
18
18
|
no session cookies, and proxy credentials are redacted from inspection and diagnostics.
|
|
19
19
|
- External payloads are parsed as `unknown` and narrowed before use.
|
|
20
|
+
- Remote manifests are untrusted. GraphQL endpoints must be HTTPS on an allowed X origin and a
|
|
21
|
+
read-only GraphQL path before session cookies, CSRF, or bearer tokens are attached. Authenticated
|
|
22
|
+
requests do not follow redirects. Tests may add synthetic origins through `allowedManifestOrigins`.
|
|
20
23
|
|
|
21
24
|
## Local state
|
|
22
25
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "xtrawl",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Scrape public X posts, profiles, followers, and following",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
@@ -62,6 +62,7 @@
|
|
|
62
62
|
"package:check": "node scripts/package-check.mjs",
|
|
63
63
|
"harness:validate": "python3 scripts/harness/validate.py",
|
|
64
64
|
"harness:budget": "python3 scripts/harness/budget.py",
|
|
65
|
+
"bench": "npm run build && node scripts/bench.mjs",
|
|
65
66
|
"prepack": "npm run build",
|
|
66
67
|
"prepublishOnly": "npm run verify",
|
|
67
68
|
"prepare": "husky"
|
|
@@ -76,13 +77,13 @@
|
|
|
76
77
|
},
|
|
77
78
|
"devDependencies": {
|
|
78
79
|
"@eslint/js": "^10.0.1",
|
|
79
|
-
"@types/jest": "^
|
|
80
|
-
"@types/node": "^
|
|
80
|
+
"@types/jest": "^30.0.0",
|
|
81
|
+
"@types/node": "^26.4.0",
|
|
81
82
|
"@typescript-eslint/eslint-plugin": "^8.62.1",
|
|
82
83
|
"@typescript-eslint/parser": "^8.62.1",
|
|
83
84
|
"eslint": "^10.6.0",
|
|
84
85
|
"husky": "^9.1.7",
|
|
85
|
-
"jest": "^
|
|
86
|
+
"jest": "^30.5.0",
|
|
86
87
|
"prettier": "^3.9.4",
|
|
87
88
|
"ts-jest": "^29.3.4",
|
|
88
89
|
"typescript": "^5.9.3"
|