scrapely 1.2.34 → 1.2.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to this project will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## [1.2.35] - 2026-10-20
|
|
6
|
+
|
|
7
|
+
- The storage client percent-encodes request ids and key-value store record keys as one URL path segment and refuses `''`, `.` and `..`. A request id such as `..` read from a queue used to send the call to another resource.
|
|
8
|
+
|
|
5
9
|
## [1.2.34] - 2026-10-19
|
|
6
10
|
|
|
7
11
|
- Initial public release.
|
package/dist/actor.js
CHANGED
|
@@ -1330,12 +1330,10 @@ class Actor {
|
|
|
1330
1330
|
if (!apiUrl || !token || !runId) {
|
|
1331
1331
|
throw new Error('Metamorph requires apiBaseUrl, token, and actorRunId in configuration');
|
|
1332
1332
|
}
|
|
1333
|
-
// Wire format (server contract, Python SDK parity): the
|
|
1334
|
-
//
|
|
1335
|
-
//
|
|
1336
|
-
//
|
|
1337
|
-
// the contract — the old client-side KVS write made the server overwrite
|
|
1338
|
-
// our value with the stale INPUT, silently losing the new input.
|
|
1333
|
+
// Wire format (server contract, Python SDK parity): the new input (if
|
|
1334
|
+
// any) goes in the request body's `input` field; the server stores it as
|
|
1335
|
+
// INPUT-METAMORPH-<n> (INPUT keeps the original) and tracks the metamorph
|
|
1336
|
+
// count in run stats. `inputKey` is NOT part of the contract.
|
|
1339
1337
|
const body = { targetActorId };
|
|
1340
1338
|
if (options?.build) {
|
|
1341
1339
|
body.build = options.build;
|
|
@@ -11,6 +11,7 @@ exports.fetchWithRetries = fetchWithRetries;
|
|
|
11
11
|
const node_crypto_1 = require("node:crypto");
|
|
12
12
|
const charging_js_1 = require("../charging.js");
|
|
13
13
|
const kvs_iteration_js_1 = require("./kvs-iteration.js");
|
|
14
|
+
const utils_js_1 = require("./utils.js");
|
|
14
15
|
function buildRqQueryParams(baseClientKey, options) {
|
|
15
16
|
const params = new URLSearchParams();
|
|
16
17
|
params.set('clientKey', options?.clientKey ?? baseClientKey);
|
|
@@ -734,11 +735,11 @@ class ApiKeyValueStoreClient extends BaseApiClient {
|
|
|
734
735
|
await this.request('DELETE', `/key-value-stores/${this.id}`);
|
|
735
736
|
}
|
|
736
737
|
async recordExists(key) {
|
|
737
|
-
return this.exists(`/key-value-stores/${this.id}/records/${key}`);
|
|
738
|
+
return this.exists(`/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`);
|
|
738
739
|
}
|
|
739
740
|
async getRecord(key, options) {
|
|
740
741
|
try {
|
|
741
|
-
const { data, contentType } = await this.requestWithHeaders('GET', `/key-value-stores/${this.id}/records/${key}`);
|
|
742
|
+
const { data, contentType } = await this.requestWithHeaders('GET', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`);
|
|
742
743
|
if (data === undefined || data === null)
|
|
743
744
|
return undefined;
|
|
744
745
|
let value = data;
|
|
@@ -791,10 +792,10 @@ class ApiKeyValueStoreClient extends BaseApiClient {
|
|
|
791
792
|
timeoutSecs: options.timeoutSecs,
|
|
792
793
|
doNotRetryTimeouts: options.doNotRetryTimeouts,
|
|
793
794
|
} : undefined;
|
|
794
|
-
await this.request('PUT', `/key-value-stores/${this.id}/records/${key}`, body, contentType || ct, requestOptions);
|
|
795
|
+
await this.request('PUT', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`, body, contentType || ct, requestOptions);
|
|
795
796
|
}
|
|
796
797
|
async deleteRecord(key) {
|
|
797
|
-
await this.request('DELETE', `/key-value-stores/${this.id}/records/${key}`);
|
|
798
|
+
await this.request('DELETE', `/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`);
|
|
798
799
|
}
|
|
799
800
|
async listKeys(options) {
|
|
800
801
|
const params = new URLSearchParams();
|
|
@@ -853,7 +854,7 @@ class ApiKeyValueStoreClient extends BaseApiClient {
|
|
|
853
854
|
// apiUrl is the container-internal API address; links handed to users
|
|
854
855
|
// must use the public base URL the platform injects.
|
|
855
856
|
const publicBase = (process.env.APIFY_API_PUBLIC_BASE_URL || process.env.ACTOR_API_PUBLIC_BASE_URL || this.config.apiUrl).replace(/\/+$/, '');
|
|
856
|
-
return `${publicBase}/v2/key-value-stores/${this.id}/records/${key}`;
|
|
857
|
+
return `${publicBase}/v2/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`;
|
|
857
858
|
}
|
|
858
859
|
}
|
|
859
860
|
exports.ApiKeyValueStoreClient = ApiKeyValueStoreClient;
|
|
@@ -920,7 +921,7 @@ class ApiRequestQueueClient extends BaseApiClient {
|
|
|
920
921
|
}
|
|
921
922
|
async getRequest(id) {
|
|
922
923
|
try {
|
|
923
|
-
return await this.request('GET', `/request-queues/${this.id}/requests/${id}`);
|
|
924
|
+
return await this.request('GET', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}`);
|
|
924
925
|
}
|
|
925
926
|
catch {
|
|
926
927
|
return undefined;
|
|
@@ -928,10 +929,10 @@ class ApiRequestQueueClient extends BaseApiClient {
|
|
|
928
929
|
}
|
|
929
930
|
async updateRequest(request, options) {
|
|
930
931
|
const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
|
|
931
|
-
return this.request('PUT', `/request-queues/${this.id}/requests/${request.id}${query}`, request);
|
|
932
|
+
return this.request('PUT', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(request.id)}${query}`, request);
|
|
932
933
|
}
|
|
933
934
|
async deleteRequest(id) {
|
|
934
|
-
await this.request('DELETE', `/request-queues/${this.id}/requests/${id}`);
|
|
935
|
+
await this.request('DELETE', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}`);
|
|
935
936
|
}
|
|
936
937
|
async listHead(options) {
|
|
937
938
|
const query = buildRqQueryParams(this.clientKey, { limit: options?.limit });
|
|
@@ -959,12 +960,12 @@ class ApiRequestQueueClient extends BaseApiClient {
|
|
|
959
960
|
lockSecs: options.lockSecs,
|
|
960
961
|
forefront: options.forefront,
|
|
961
962
|
});
|
|
962
|
-
const result = await this.request('PUT', `/request-queues/${this.id}/requests/${id}/lock${query}`);
|
|
963
|
+
const result = await this.request('PUT', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}/lock${query}`);
|
|
963
964
|
return { lockExpiresAt: new Date(result.lockExpiresAt) };
|
|
964
965
|
}
|
|
965
966
|
async deleteRequestLock(id, options) {
|
|
966
967
|
const query = buildRqQueryParams(this.clientKey, { forefront: options?.forefront });
|
|
967
|
-
await this.request('DELETE', `/request-queues/${this.id}/requests/${id}/lock${query}`);
|
|
968
|
+
await this.request('DELETE', `/request-queues/${this.id}/requests/${(0, utils_js_1.pathSegment)(id)}/lock${query}`);
|
|
968
969
|
}
|
|
969
970
|
/**
|
|
970
971
|
* List requests in the queue with optional filtering and pagination.
|
|
@@ -297,7 +297,7 @@ class DragonflyKeyValueStoreClient {
|
|
|
297
297
|
getPublicUrl(key) {
|
|
298
298
|
const config = configuration_js_1.Configuration.getGlobalConfig();
|
|
299
299
|
const baseUrl = config.get('apiPublicBaseUrl') || config.get('apiBaseUrl') || 'https://api.scrape.ly';
|
|
300
|
-
return `${baseUrl}/v2/key-value-stores/${this.id}/records/${key}`;
|
|
300
|
+
return `${baseUrl}/v2/key-value-stores/${this.id}/records/${(0, utils_js_1.pathSegment)(key)}`;
|
|
301
301
|
}
|
|
302
302
|
}
|
|
303
303
|
exports.DragonflyKeyValueStoreClient = DragonflyKeyValueStoreClient;
|
package/dist/storage/utils.d.ts
CHANGED
|
@@ -1,3 +1,11 @@
|
|
|
1
1
|
export declare function uniqueKeyToRequestId(uniqueKey: string): string;
|
|
2
2
|
export declare function generateId(): string;
|
|
3
3
|
export declare function nowISO(): Date;
|
|
4
|
+
/**
|
|
5
|
+
* Percent-encode a value so it stays one URL path segment. Request ids come
|
|
6
|
+
* from queue contents another client may have written: unencoded, an id such
|
|
7
|
+
* as `..` or `../../key-value-stores/x` would send the call to another
|
|
8
|
+
* resource. `''`, `.` and `..` are rejected because a URL parser resolves dot
|
|
9
|
+
* segments even when they are percent-encoded.
|
|
10
|
+
*/
|
|
11
|
+
export declare function pathSegment(value: string): string;
|
package/dist/storage/utils.js
CHANGED
|
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
3
3
|
exports.uniqueKeyToRequestId = uniqueKeyToRequestId;
|
|
4
4
|
exports.generateId = generateId;
|
|
5
5
|
exports.nowISO = nowISO;
|
|
6
|
+
exports.pathSegment = pathSegment;
|
|
6
7
|
const node_crypto_1 = require("node:crypto");
|
|
7
8
|
function uniqueKeyToRequestId(uniqueKey) {
|
|
8
9
|
return (0, node_crypto_1.createHash)('sha256')
|
|
@@ -17,3 +18,19 @@ function generateId() {
|
|
|
17
18
|
function nowISO() {
|
|
18
19
|
return new Date();
|
|
19
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* Percent-encode a value so it stays one URL path segment. Request ids come
|
|
23
|
+
* from queue contents another client may have written: unencoded, an id such
|
|
24
|
+
* as `..` or `../../key-value-stores/x` would send the call to another
|
|
25
|
+
* resource. `''`, `.` and `..` are rejected because a URL parser resolves dot
|
|
26
|
+
* segments even when they are percent-encoded.
|
|
27
|
+
*/
|
|
28
|
+
function pathSegment(value) {
|
|
29
|
+
if (typeof value !== 'string') {
|
|
30
|
+
throw new TypeError(`Expected a string for a URL path segment, got ${typeof value}.`);
|
|
31
|
+
}
|
|
32
|
+
if (value === '' || value === '.' || value === '..') {
|
|
33
|
+
throw new Error(`"${value}" cannot be used as a URL path segment.`);
|
|
34
|
+
}
|
|
35
|
+
return encodeURIComponent(value);
|
|
36
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "scrapely",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.35",
|
|
4
4
|
"description": "Scrapely SDK for building JavaScript and TypeScript Actors",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -47,6 +47,7 @@
|
|
|
47
47
|
"test:charge-pending-persist": "npx tsx --test tests/charge-pending-persist.test.ts",
|
|
48
48
|
"test:charge-platform-budget": "npx tsx --test tests/charge-platform-budget.test.ts",
|
|
49
49
|
"test:storage-retries": "npx tsx --test tests/storage-retries.test.ts",
|
|
50
|
+
"test:storage-path-segments": "npx tsx --test tests/storage-path-segments.test.ts",
|
|
50
51
|
"test:push-idempotency": "npx tsx --test tests/push-idempotency.test.ts",
|
|
51
52
|
"test:stdio-drain": "npx tsx --test tests/stdio-drain.test.ts",
|
|
52
53
|
"test:charging-client": "npx tsx --test tests/charging-client.test.ts",
|