scrapely 1.2.3 → 1.2.5

package/dist/storage/api-storage-client.d.ts

@@ -0,0 +1,199 @@
+import type { DatasetClient, DatasetInfo, DatasetClientListOptions, DatasetClientUpdateOptions, KeyValueStoreClient, KeyValueStoreInfo, KeyValueStoreRecord, KeyValueStoreClientGetRecordOptions, KeyValueStoreClientListOptions, KeyValueStoreClientListData, KeyValueStoreRecordOptions, KeyValueStoreClientUpdateOptions, RequestQueueClient, RequestQueueInfo, QueueHead, QueueOperationInfo, RequestSchema, UpdateRequestSchema, RequestOptions, BatchAddRequestsResult, ListOptions, ListAndLockOptions, ListAndLockHeadResult, ProlongRequestLockOptions, ProlongRequestLockResult, DeleteRequestLockOptions, PaginatedList, Dictionary, StorageClient, RequestQueueOptions } from '@crawlee/types';
+import type { ListRequestsOptions, ListRequestsResult } from './request-queue-client.js';
+/**
+ * Configuration for API-based storage client
+ */
+export interface ApiStorageConfig {
+    apiUrl: string;
+    token: string;
+    runId: string;
+}
+export declare class ApiStorageClientError extends Error {
+    readonly statusCode: number;
+    constructor(message: string, statusCode: number);
+}
+/**
+ * Options for API requests (passed through from RecordOptions)
+ */
+interface ApiRequestOptions {
+    /** Timeout in seconds for the request */
+    timeoutSecs?: number;
+    /** If true, don't retry on timeout errors */
+    doNotRetryTimeouts?: boolean;
+}
+/**
+ * Base class for API-based storage clients.
+ * Communicates with internal storage API instead of direct DB access.
+ * No sensitive credentials are stored - only the user's API key.
+ */
+declare abstract class BaseApiClient {
+    protected config: ApiStorageConfig;
+    protected headers: Record<string, string>;
+    private static rateLimiter;
+    private static maxRetries;
+    private static defaultTimeoutMs;
+    constructor(config: ApiStorageConfig);
+    protected request<T>(method: string, path: string, body?: any, contentType?: string, options?: ApiRequestOptions): Promise<T>;
+    protected requestWithHeaders<T>(method: string, path: string, body?: any, contentType?: string): Promise<{
+        data: T;
+        contentType: string;
+    }>;
+}
+/**
+ * Extended options for listing dataset items with filtering support.
+ * Matches Crawlee's DatasetDataOptions interface.
+ */
+interface ExtendedDatasetListOptions extends DatasetClientListOptions {
+    /** An array of field names that will be included in the result. */
+    fields?: string[];
+    /** If true then the function returns only non-empty items and skips hidden fields. */
+    clean?: boolean;
+    /** If true then the function doesn't return hidden fields (fields starting with "#"). */
+    skipHidden?: boolean;
+    /** If true then the function doesn't return empty items. */
+    skipEmpty?: boolean;
+}
+/**
+ * Dataset client that communicates via internal API
+ */
+export declare class ApiDatasetClient<Data extends Dictionary = Dictionary> extends BaseApiClient implements DatasetClient<Data> {
+    private id;
+    constructor(config: ApiStorageConfig, datasetId: string);
+    get(): Promise<DatasetInfo | undefined>;
+    update(newFields: DatasetClientUpdateOptions): Promise<Partial<DatasetInfo>>;
+    delete(): Promise<void>;
+    downloadItems(): Promise<Buffer>;
+    listItems(options?: ExtendedDatasetListOptions): AsyncIterable<Data> & Promise<PaginatedList<Data>>;
+    /**
+     * Iterates over dataset entries (index-value pairs) using an async generator.
+     * Required by Crawlee's Dataset.entries() method.
+     */
+    listEntries(options?: ExtendedDatasetListOptions): AsyncIterable<[number, Data]> & Promise<PaginatedList<[number, Data]>>;
+    private _listItems;
+    pushItems(items: Data | Data[] | string | string[]): Promise<void>;
+}
+/**
+ * Key-Value Store client that communicates via internal API
+ */
+export declare class ApiKeyValueStoreClient extends BaseApiClient implements KeyValueStoreClient {
+    private id;
+    constructor(config: ApiStorageConfig, storeId: string);
+    get(): Promise<KeyValueStoreInfo | undefined>;
+    update(newFields: KeyValueStoreClientUpdateOptions): Promise<Partial<KeyValueStoreInfo>>;
+    delete(): Promise<void>;
+    recordExists(key: string): Promise<boolean>;
+    getRecord(key: string, options?: KeyValueStoreClientGetRecordOptions): Promise<KeyValueStoreRecord | undefined>;
+    setRecord(record: KeyValueStoreRecord, options?: KeyValueStoreRecordOptions): Promise<void>;
+    deleteRecord(key: string): Promise<void>;
+    listKeys(options?: KeyValueStoreClientListOptions): Promise<KeyValueStoreClientListData>;
+    keys(options?: KeyValueStoreClientListOptions): AsyncIterable<string> & Promise<KeyValueStoreClientListData>;
+    values(options?: KeyValueStoreClientListOptions): AsyncIterable<unknown> & Promise<unknown[]>;
+    entries(options?: KeyValueStoreClientListOptions): AsyncIterable<[string, unknown]> & Promise<[string, unknown][]>;
+    /**
+     * Iterates over all keys in the key-value store, calling the iteratee function for each key.
+     * Similar to Apify's KeyValueStore.forEachKey() method.
+     */
+    forEachKey(iteratee: (key: string, index: number, info: {
+        size: number;
+    }) => Promise<void> | void, options?: KeyValueStoreClientListOptions): Promise<void>;
+    /**
+     * Gets a value from the key-value store that is automatically saved when modified.
+     * If the key doesn't exist, returns the defaultValue and stores it for future use.
+     * Similar to Apify's KeyValueStore.getAutoSavedValue() method.
+     *
+     * @param key - The key of the record to get
+     * @param defaultValue - The default value to return and store if the key doesn't exist
+     * @returns The stored value or the defaultValue
+     */
+    getAutoSavedValue<T>(key: string, defaultValue: T): Promise<T>;
+    /**
+     * Returns a URL for the given key that may be used to publicly access the value
+     * in the remote key-value store.
+     * Similar to Apify's KeyValueStore.getPublicUrl() method.
+     *
+     * @param key - The key of the record to get the public URL for
+     * @returns A public URL for accessing the record
+     */
+    getPublicUrl(key: string): string;
+}
+/**
+ * Request Queue client that communicates via internal API
+ */
+export declare class ApiRequestQueueClient extends BaseApiClient implements RequestQueueClient {
+    private id;
+    private readonly clientKey;
+    constructor(config: ApiStorageConfig, queueId: string, options?: RequestQueueOptions);
+    get(): Promise<RequestQueueInfo | undefined>;
+    update(newFields: {
+        name?: string;
+    }): Promise<Partial<RequestQueueInfo> | undefined>;
+    delete(): Promise<void>;
+    addRequest(request: RequestSchema, options?: RequestOptions): Promise<QueueOperationInfo>;
+    batchAddRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
+    getRequest(id: string): Promise<RequestOptions | undefined>;
+    updateRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo>;
+    deleteRequest(id: string): Promise<void>;
+    listHead(options?: ListOptions): Promise<QueueHead>;
+    listAndLockHead(options: ListAndLockOptions): Promise<ListAndLockHeadResult>;
+    prolongRequestLock(id: string, options: ProlongRequestLockOptions): Promise<ProlongRequestLockResult>;
+    deleteRequestLock(id: string, options?: DeleteRequestLockOptions): Promise<void>;
+    /**
+     * List requests in the queue with optional filtering and pagination (Apify REST API).
+     */
+    listRequests(options?: ListRequestsOptions): Promise<ListRequestsResult>;
+    /**
+     * Unlock all requests locked by this clientKey (Apify REST API).
+     */
+    unlockRequests(options: {
+        clientKey?: string;
+        forefront?: boolean;
+        runId?: string;
+    }): Promise<{
+        unlockedCount: number;
+    }>;
+}
+/**
+ * Dataset collection client (API-based)
+ */
+declare class ApiDatasetCollectionClient {
+    private config;
+    constructor(config: ApiStorageConfig);
+    getOrCreate(name?: string): Promise<DatasetInfo>;
+    list(): Promise<PaginatedList<DatasetInfo>>;
+}
+/**
+ * Key-value store collection client (API-based)
+ */
+declare class ApiKeyValueStoreCollectionClient {
+    private config;
+    constructor(config: ApiStorageConfig);
+    getOrCreate(name?: string): Promise<KeyValueStoreInfo>;
+    list(): Promise<PaginatedList<KeyValueStoreInfo>>;
+}
+/**
+ * Request queue collection client (API-based)
+ */
+declare class ApiRequestQueueCollectionClient {
+    private config;
+    constructor(config: ApiStorageConfig);
+    getOrCreate(name?: string): Promise<RequestQueueInfo>;
+    list(): Promise<PaginatedList<RequestQueueInfo>>;
+}
+/**
+ * Factory for creating API-based storage clients.
+ * Implements StorageClient interface for Crawlee compatibility.
+ */
+export declare class ApiStorageClientFactory implements StorageClient {
+    private config;
+    constructor(config: ApiStorageConfig);
+    datasets(): ApiDatasetCollectionClient;
+    dataset<Data extends Dictionary = Dictionary>(id: string): DatasetClient<Data>;
+    keyValueStores(): ApiKeyValueStoreCollectionClient;
+    keyValueStore(id: string): KeyValueStoreClient;
+    requestQueues(): ApiRequestQueueCollectionClient;
+    requestQueue(id: string, options?: RequestQueueOptions): RequestQueueClient;
+    purge(): Promise<void>;
+    teardown(): Promise<void>;
+    setStatusMessage(message: string): Promise<void>;
+}
+export {};