s3db.js 18.0.11-next.1534f717 → 18.0.11-next.47047b5d
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/clients/recker-http-handler.js +56 -8
- package/dist/clients/recker-http-handler.js.map +1 -1
- package/dist/concerns/id/alphabets.js +150 -0
- package/dist/concerns/id/alphabets.js.map +1 -0
- package/dist/concerns/id/entropy.js +243 -0
- package/dist/concerns/id/entropy.js.map +1 -0
- package/dist/concerns/id/generators/nanoid.js +74 -0
- package/dist/concerns/id/generators/nanoid.js.map +1 -0
- package/dist/concerns/id/generators/sid.js +73 -0
- package/dist/concerns/id/generators/sid.js.map +1 -0
- package/dist/concerns/id/generators/ulid.js +208 -0
- package/dist/concerns/id/generators/ulid.js.map +1 -0
- package/dist/concerns/id/generators/uuid-v7.js +150 -0
- package/dist/concerns/id/generators/uuid-v7.js.map +1 -0
- package/dist/concerns/id/index.js +74 -0
- package/dist/concerns/id/index.js.map +1 -0
- package/dist/concerns/plugin-storage.js +114 -0
- package/dist/concerns/plugin-storage.js.map +1 -1
- package/dist/concerns/s3-errors.js +72 -0
- package/dist/concerns/s3-errors.js.map +1 -0
- package/dist/concerns/s3-key.js +54 -0
- package/dist/concerns/s3-key.js.map +1 -0
- package/dist/concerns/safe-merge.js +47 -0
- package/dist/concerns/safe-merge.js.map +1 -0
- package/dist/core/resource-config-validator.js +12 -2
- package/dist/core/resource-config-validator.js.map +1 -1
- package/dist/core/resource-partitions.class.js +12 -1
- package/dist/core/resource-partitions.class.js.map +1 -1
- package/dist/core/resource-persistence.class.js +41 -12
- package/dist/core/resource-persistence.class.js.map +1 -1
- package/dist/core/resource-query.class.js +21 -47
- package/dist/core/resource-query.class.js.map +1 -1
- package/dist/plugins/concerns/s3-mutex.class.js +155 -0
- package/dist/plugins/concerns/s3-mutex.class.js.map +1 -0
- package/dist/plugins/recon/stages/recker-asn-stage.js +279 -0
- package/dist/plugins/recon/stages/recker-asn-stage.js.map +1 -0
- package/dist/plugins/recon/stages/recker-dns-stage.js +227 -0
- package/dist/plugins/recon/stages/recker-dns-stage.js.map +1 -0
- package/dist/plugins/recon/stages/recker-scrape-stage.js +369 -0
- package/dist/plugins/recon/stages/recker-scrape-stage.js.map +1 -0
- package/dist/plugins/spider/recker-link-discoverer.js +544 -0
- package/dist/plugins/spider/recker-link-discoverer.js.map +1 -0
- package/dist/plugins/spider/recker-llms-validator.js +334 -0
- package/dist/plugins/spider/recker-llms-validator.js.map +1 -0
- package/dist/plugins/spider/recker-robots-validator.js +336 -0
- package/dist/plugins/spider/recker-robots-validator.js.map +1 -0
- package/dist/plugins/spider/recker-security-adapter.js +325 -0
- package/dist/plugins/spider/recker-security-adapter.js.map +1 -0
- package/dist/plugins/spider/recker-seo-adapter.js +399 -0
- package/dist/plugins/spider/recker-seo-adapter.js.map +1 -0
- package/dist/plugins/spider/recker-sitemap-validator.js +406 -0
- package/dist/plugins/spider/recker-sitemap-validator.js.map +1 -0
- package/dist/resource.class.js +2 -0
- package/dist/resource.class.js.map +1 -1
- package/dist/s3db.cjs +353 -71
- package/dist/s3db.cjs.map +1 -1
- package/dist/s3db.es.js +354 -72
- package/dist/s3db.es.js.map +1 -1
- package/dist/types/clients/recker-http-handler.d.ts +1 -0
- package/dist/types/clients/recker-http-handler.d.ts.map +1 -1
- package/dist/types/clients/types.d.ts +14 -0
- package/dist/types/clients/types.d.ts.map +1 -1
- package/dist/types/concerns/id/alphabets.d.ts +125 -0
- package/dist/types/concerns/id/alphabets.d.ts.map +1 -0
- package/dist/types/concerns/id/entropy.d.ts +84 -0
- package/dist/types/concerns/id/entropy.d.ts.map +1 -0
- package/dist/types/concerns/id/generators/nanoid.d.ts +46 -0
- package/dist/types/concerns/id/generators/nanoid.d.ts.map +1 -0
- package/dist/types/concerns/id/generators/sid.d.ts +45 -0
- package/dist/types/concerns/id/generators/sid.d.ts.map +1 -0
- package/dist/types/concerns/id/generators/ulid.d.ts +71 -0
- package/dist/types/concerns/id/generators/ulid.d.ts.map +1 -0
- package/dist/types/concerns/id/generators/uuid-v7.d.ts +60 -0
- package/dist/types/concerns/id/generators/uuid-v7.d.ts.map +1 -0
- package/dist/types/concerns/id/index.d.ts +51 -0
- package/dist/types/concerns/id/index.d.ts.map +1 -0
- package/dist/types/concerns/plugin-storage.d.ts +25 -0
- package/dist/types/concerns/plugin-storage.d.ts.map +1 -1
- package/dist/types/concerns/s3-errors.d.ts +20 -0
- package/dist/types/concerns/s3-errors.d.ts.map +1 -0
- package/dist/types/concerns/s3-key.d.ts +30 -0
- package/dist/types/concerns/s3-key.d.ts.map +1 -0
- package/dist/types/concerns/safe-merge.d.ts +22 -0
- package/dist/types/concerns/safe-merge.d.ts.map +1 -0
- package/dist/types/core/resource-config-validator.d.ts.map +1 -1
- package/dist/types/core/resource-partitions.class.d.ts.map +1 -1
- package/dist/types/core/resource-persistence.class.d.ts.map +1 -1
- package/dist/types/core/resource-query.class.d.ts.map +1 -1
- package/dist/types/plugins/concerns/s3-mutex.class.d.ts +30 -0
- package/dist/types/plugins/concerns/s3-mutex.class.d.ts.map +1 -0
- package/dist/types/plugins/recon/stages/recker-asn-stage.d.ts +90 -0
- package/dist/types/plugins/recon/stages/recker-asn-stage.d.ts.map +1 -0
- package/dist/types/plugins/recon/stages/recker-dns-stage.d.ts +125 -0
- package/dist/types/plugins/recon/stages/recker-dns-stage.d.ts.map +1 -0
- package/dist/types/plugins/recon/stages/recker-scrape-stage.d.ts +96 -0
- package/dist/types/plugins/recon/stages/recker-scrape-stage.d.ts.map +1 -0
- package/dist/types/plugins/spider/recker-link-discoverer.d.ts +54 -0
- package/dist/types/plugins/spider/recker-link-discoverer.d.ts.map +1 -0
- package/dist/types/plugins/spider/recker-llms-validator.d.ts +105 -0
- package/dist/types/plugins/spider/recker-llms-validator.d.ts.map +1 -0
- package/dist/types/plugins/spider/recker-robots-validator.d.ts +92 -0
- package/dist/types/plugins/spider/recker-robots-validator.d.ts.map +1 -0
- package/dist/types/plugins/spider/recker-security-adapter.d.ts +83 -0
- package/dist/types/plugins/spider/recker-security-adapter.d.ts.map +1 -0
- package/dist/types/plugins/spider/recker-seo-adapter.d.ts +187 -0
- package/dist/types/plugins/spider/recker-seo-adapter.d.ts.map +1 -0
- package/dist/types/plugins/spider/recker-sitemap-validator.d.ts +121 -0
- package/dist/types/plugins/spider/recker-sitemap-validator.d.ts.map +1 -0
- package/dist/types/resource.class.d.ts.map +1 -1
- package/mcp/prompts/index.ts +275 -0
- package/mcp/resources/index.ts +322 -0
- package/mcp/tools/plugins.ts +1137 -0
- package/mcp/tools/streams.ts +340 -0
- package/package.json +20 -21
- package/src/clients/recker-http-handler.ts +74 -8
- package/src/clients/types.ts +14 -0
- package/src/concerns/id/alphabets.ts +175 -0
- package/src/concerns/id/entropy.ts +286 -0
- package/src/concerns/id/generators/sid.ts +90 -0
- package/src/concerns/id/generators/ulid.ts +249 -0
- package/src/concerns/id/generators/uuid-v7.ts +179 -0
- package/src/concerns/id/index.ts +167 -0
- package/src/concerns/plugin-storage.ts +144 -0
- package/src/concerns/s3-errors.ts +97 -0
- package/src/concerns/s3-key.ts +62 -0
- package/src/concerns/safe-merge.ts +60 -0
- package/src/core/resource-config-validator.ts +9 -2
- package/src/core/resource-partitions.class.ts +14 -1
- package/src/core/resource-persistence.class.ts +47 -13
- package/src/core/resource-query.class.ts +21 -46
- package/src/plugins/concerns/s3-mutex.class.ts +228 -0
- package/src/plugins/recon/stages/recker-asn-stage.ts +385 -0
- package/src/plugins/recon/stages/recker-dns-stage.ts +360 -0
- package/src/plugins/recon/stages/recker-scrape-stage.ts +509 -0
- package/src/plugins/spider/recker-link-discoverer.ts +645 -0
- package/src/plugins/spider/recker-llms-validator.ts +500 -0
- package/src/plugins/spider/recker-robots-validator.ts +473 -0
- package/src/plugins/spider/recker-security-adapter.ts +489 -0
- package/src/plugins/spider/recker-seo-adapter.ts +605 -0
- package/src/plugins/spider/recker-sitemap-validator.ts +621 -0
- package/src/resource.class.ts +2 -0
|
@@ -597,6 +597,150 @@ export class PluginStorage {
|
|
|
597
597
|
return Promise.all(promises);
|
|
598
598
|
}
|
|
599
599
|
|
|
600
|
+
/**
|
|
601
|
+
* Set data only if the key does not exist (conditional PUT).
|
|
602
|
+
* Uses ifNoneMatch: '*' to ensure atomicity.
|
|
603
|
+
* @returns The ETag (version) if set succeeded, null if key already exists.
|
|
604
|
+
*/
|
|
605
|
+
async setIfNotExists(key: string, data: Record<string, unknown>, options: PluginStorageSetOptions = {}): Promise<string | null> {
|
|
606
|
+
const [ok, err, response] = await tryFn(() => this.set(key, data, { ...options, ifNoneMatch: '*' }));
|
|
607
|
+
|
|
608
|
+
if (!ok) {
|
|
609
|
+
const error = err as { name?: string; code?: string; statusCode?: number } | undefined;
|
|
610
|
+
// PreconditionFailed (412) or similar means key already exists
|
|
611
|
+
if (
|
|
612
|
+
error?.name === 'PreconditionFailed' ||
|
|
613
|
+
error?.code === 'PreconditionFailed' ||
|
|
614
|
+
error?.statusCode === 412
|
|
615
|
+
) {
|
|
616
|
+
return null;
|
|
617
|
+
}
|
|
618
|
+
throw err;
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
return response?.ETag ?? null;
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* Get data along with its version (ETag) for conditional updates.
|
|
626
|
+
* @returns Object with data and version, or { data: null, version: null } if not found.
|
|
627
|
+
*/
|
|
628
|
+
async getWithVersion(key: string): Promise<{ data: Record<string, unknown> | null; version: string | null }> {
|
|
629
|
+
const [ok, err, response] = await tryFn<GetObjectResponse>(() => this.client.getObject(key));
|
|
630
|
+
|
|
631
|
+
if (!ok || !response) {
|
|
632
|
+
const error = err as { name?: string; code?: string; Code?: string; statusCode?: number } | undefined;
|
|
633
|
+
if (
|
|
634
|
+
error?.name === 'NoSuchKey' ||
|
|
635
|
+
error?.code === 'NoSuchKey' ||
|
|
636
|
+
error?.Code === 'NoSuchKey' ||
|
|
637
|
+
error?.statusCode === 404
|
|
638
|
+
) {
|
|
639
|
+
return { data: null, version: null };
|
|
640
|
+
}
|
|
641
|
+
throw new PluginStorageError(`Failed to retrieve plugin data with version`, {
|
|
642
|
+
pluginSlug: this.pluginSlug,
|
|
643
|
+
key,
|
|
644
|
+
operation: 'getWithVersion',
|
|
645
|
+
original: err,
|
|
646
|
+
suggestion: 'Check if the key exists and S3 permissions are correct'
|
|
647
|
+
});
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
const metadata = response.Metadata || {};
|
|
651
|
+
const parsedMetadata = this._parseMetadataValues(metadata);
|
|
652
|
+
let data: Record<string, unknown> = parsedMetadata;
|
|
653
|
+
|
|
654
|
+
if (response.Body) {
|
|
655
|
+
const [parseOk, parseErr, result] = await tryFn(async () => {
|
|
656
|
+
const bodyContent = await response.Body!.transformToString();
|
|
657
|
+
if (bodyContent && bodyContent.trim()) {
|
|
658
|
+
const body = JSON.parse(bodyContent);
|
|
659
|
+
return { ...parsedMetadata, ...body };
|
|
660
|
+
}
|
|
661
|
+
return parsedMetadata;
|
|
662
|
+
});
|
|
663
|
+
|
|
664
|
+
if (!parseOk || !result) {
|
|
665
|
+
throw new PluginStorageError(`Failed to parse JSON body`, {
|
|
666
|
+
pluginSlug: this.pluginSlug,
|
|
667
|
+
key,
|
|
668
|
+
operation: 'getWithVersion',
|
|
669
|
+
original: parseErr,
|
|
670
|
+
suggestion: 'Body content may be corrupted'
|
|
671
|
+
});
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
data = result;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
// Check expiration
|
|
678
|
+
const expiresAt = (data._expiresat || data._expiresAt) as number | undefined;
|
|
679
|
+
if (expiresAt && Date.now() > expiresAt) {
|
|
680
|
+
await this.delete(key);
|
|
681
|
+
return { data: null, version: null };
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
// Clean up internal fields
|
|
685
|
+
delete data._expiresat;
|
|
686
|
+
delete data._expiresAt;
|
|
687
|
+
|
|
688
|
+
// Extract ETag from response - need to get it from headObject since getObject may not return it
|
|
689
|
+
const [headOk, , headResponse] = await tryFn<HeadObjectResponse & { ETag?: string }>(() =>
|
|
690
|
+
this.client.headObject(key)
|
|
691
|
+
);
|
|
692
|
+
const version = headOk && headResponse ? (headResponse as any).ETag ?? null : null;
|
|
693
|
+
|
|
694
|
+
return { data, version };
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
/**
|
|
698
|
+
* Set data only if the current version matches (conditional PUT).
|
|
699
|
+
* Uses ifMatch to ensure no concurrent modifications.
|
|
700
|
+
* @returns The new ETag (version) if set succeeded, null if version mismatch.
|
|
701
|
+
*/
|
|
702
|
+
async setIfVersion(key: string, data: Record<string, unknown>, version: string, options: PluginStorageSetOptions = {}): Promise<string | null> {
|
|
703
|
+
const [ok, err, response] = await tryFn(() => this.set(key, data, { ...options, ifMatch: version }));
|
|
704
|
+
|
|
705
|
+
if (!ok) {
|
|
706
|
+
const error = err as { name?: string; code?: string; statusCode?: number } | undefined;
|
|
707
|
+
// PreconditionFailed (412) means version mismatch
|
|
708
|
+
if (
|
|
709
|
+
error?.name === 'PreconditionFailed' ||
|
|
710
|
+
error?.code === 'PreconditionFailed' ||
|
|
711
|
+
error?.statusCode === 412
|
|
712
|
+
) {
|
|
713
|
+
return null;
|
|
714
|
+
}
|
|
715
|
+
throw err;
|
|
716
|
+
}
|
|
717
|
+
|
|
718
|
+
return response?.ETag ?? null;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
/**
|
|
722
|
+
* Delete data only if the current version matches (conditional DELETE).
|
|
723
|
+
* @returns true if deleted, false if version mismatch or key not found.
|
|
724
|
+
*/
|
|
725
|
+
async deleteIfVersion(key: string, version: string): Promise<boolean> {
|
|
726
|
+
// First verify the version matches
|
|
727
|
+
const [headOk, , headResponse] = await tryFn<HeadObjectResponse & { ETag?: string }>(() =>
|
|
728
|
+
this.client.headObject(key)
|
|
729
|
+
);
|
|
730
|
+
|
|
731
|
+
if (!headOk || !headResponse) {
|
|
732
|
+
return false;
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
const currentVersion = (headResponse as any).ETag;
|
|
736
|
+
if (currentVersion !== version) {
|
|
737
|
+
return false;
|
|
738
|
+
}
|
|
739
|
+
|
|
740
|
+
const [deleteOk] = await tryFn(() => this.client.deleteObject(key));
|
|
741
|
+
return deleteOk;
|
|
742
|
+
}
|
|
743
|
+
|
|
600
744
|
async acquireLock(lockName: string, options: AcquireOptions = {}): Promise<LockHandle | null> {
|
|
601
745
|
return this._lock.acquire(lockName, options);
|
|
602
746
|
}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* S3 Error Classification Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides consistent error classification across all S3 operations.
|
|
5
|
+
* Handles differences between AWS SDK v3, MinIO, and other S3-compatible clients.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
interface S3ErrorLike {
|
|
9
|
+
name?: string;
|
|
10
|
+
code?: string;
|
|
11
|
+
Code?: string;
|
|
12
|
+
statusCode?: number;
|
|
13
|
+
$metadata?: {
|
|
14
|
+
httpStatusCode?: number;
|
|
15
|
+
};
|
|
16
|
+
message?: string;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Checks if an error indicates the object/resource was not found.
|
|
21
|
+
* Handles various S3 client error formats (AWS SDK v3, MinIO, etc.)
|
|
22
|
+
*/
|
|
23
|
+
export function isNotFoundError(error: unknown): boolean {
|
|
24
|
+
if (!error) return false;
|
|
25
|
+
|
|
26
|
+
const err = error as S3ErrorLike;
|
|
27
|
+
|
|
28
|
+
return (
|
|
29
|
+
err.name === 'NoSuchKey' ||
|
|
30
|
+
err.name === 'NotFound' ||
|
|
31
|
+
err.code === 'NoSuchKey' ||
|
|
32
|
+
err.code === 'NotFound' ||
|
|
33
|
+
err.Code === 'NoSuchKey' ||
|
|
34
|
+
err.Code === 'NotFound' ||
|
|
35
|
+
err.statusCode === 404 ||
|
|
36
|
+
err.$metadata?.httpStatusCode === 404 ||
|
|
37
|
+
(typeof err.message === 'string' && err.message.includes('NoSuchKey'))
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Checks if an error indicates access was denied.
|
|
43
|
+
*/
|
|
44
|
+
export function isAccessDeniedError(error: unknown): boolean {
|
|
45
|
+
if (!error) return false;
|
|
46
|
+
|
|
47
|
+
const err = error as S3ErrorLike;
|
|
48
|
+
|
|
49
|
+
return (
|
|
50
|
+
err.name === 'AccessDenied' ||
|
|
51
|
+
err.code === 'AccessDenied' ||
|
|
52
|
+
err.Code === 'AccessDenied' ||
|
|
53
|
+
err.statusCode === 403 ||
|
|
54
|
+
err.$metadata?.httpStatusCode === 403
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Checks if an error is a transient/retriable error (network, timeout, etc.)
|
|
60
|
+
*/
|
|
61
|
+
export function isTransientError(error: unknown): boolean {
|
|
62
|
+
if (!error) return false;
|
|
63
|
+
|
|
64
|
+
const err = error as S3ErrorLike;
|
|
65
|
+
const statusCode = err.statusCode || err.$metadata?.httpStatusCode;
|
|
66
|
+
|
|
67
|
+
if (statusCode && statusCode >= 500 && statusCode < 600) {
|
|
68
|
+
return true;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (statusCode === 429) {
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
const retriableNames = [
|
|
76
|
+
'TimeoutError',
|
|
77
|
+
'RequestTimeout',
|
|
78
|
+
'NetworkError',
|
|
79
|
+
'ECONNRESET',
|
|
80
|
+
'ETIMEDOUT',
|
|
81
|
+
'ENOTFOUND',
|
|
82
|
+
'SocketError',
|
|
83
|
+
'ServiceUnavailable',
|
|
84
|
+
'SlowDown',
|
|
85
|
+
'ThrottlingException',
|
|
86
|
+
];
|
|
87
|
+
|
|
88
|
+
if (err.name && retriableNames.includes(err.name)) {
|
|
89
|
+
return true;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (err.code && retriableNames.includes(err.code)) {
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* S3 Key Utilities
|
|
3
|
+
*
|
|
4
|
+
* S3 keys always use POSIX-style forward slashes regardless of the operating system.
|
|
5
|
+
* These utilities ensure consistent key construction across all platforms.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { ValidationError } from '../errors.js';
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Join path segments for S3 keys using forward slashes.
|
|
12
|
+
* Unlike path.join(), this always uses '/' as separator.
|
|
13
|
+
* Also normalizes any backslashes within segments to forward slashes.
|
|
14
|
+
*/
|
|
15
|
+
export function joinS3Key(...segments: string[]): string {
|
|
16
|
+
return segments
|
|
17
|
+
.filter(s => s && s.length > 0)
|
|
18
|
+
.join('/')
|
|
19
|
+
.replace(/\\/g, '/')
|
|
20
|
+
.replace(/\/+/g, '/');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Normalize an S3 key by replacing backslashes with forward slashes.
|
|
25
|
+
* Useful when migrating keys that may have been incorrectly constructed.
|
|
26
|
+
*/
|
|
27
|
+
export function normalizeS3Key(key: string): string {
|
|
28
|
+
return key.replace(/\\/g, '/').replace(/\/+/g, '/');
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
const UNSAFE_KEY_CHARS = /[\\\/=%]/;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Validates that a value is safe for use in S3 keys.
|
|
35
|
+
* IDs and partition values must be URL-friendly (no /, \, =, or %).
|
|
36
|
+
* Returns true if valid, false if contains unsafe characters.
|
|
37
|
+
*/
|
|
38
|
+
export function isValidS3KeySegment(value: string): boolean {
|
|
39
|
+
return !UNSAFE_KEY_CHARS.test(value);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Validates a value for S3 key usage, throwing ValidationError if invalid.
|
|
44
|
+
* Use this for IDs and partition values.
|
|
45
|
+
* Accepts any value type - coerces to string for validation.
|
|
46
|
+
*/
|
|
47
|
+
export function validateS3KeySegment(value: unknown, context: string): void {
|
|
48
|
+
const strValue = String(value);
|
|
49
|
+
if (UNSAFE_KEY_CHARS.test(strValue)) {
|
|
50
|
+
const invalidChars = strValue.match(UNSAFE_KEY_CHARS);
|
|
51
|
+
throw new ValidationError(
|
|
52
|
+
`Invalid ${context}: contains unsafe character '${invalidChars?.[0]}'`,
|
|
53
|
+
{
|
|
54
|
+
field: context,
|
|
55
|
+
value: strValue,
|
|
56
|
+
constraint: 'url-safe',
|
|
57
|
+
statusCode: 400,
|
|
58
|
+
suggestion: 'IDs and partition values must be URL-friendly (no /, \\, =, or %). Use alphanumeric characters, hyphens, or underscores.'
|
|
59
|
+
}
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Safe Merge Utilities
|
|
3
|
+
*
|
|
4
|
+
* Provides functions to sanitize object keys before merging,
|
|
5
|
+
* preventing prototype pollution attacks via __proto__, constructor, or prototype keys.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
const DANGEROUS_KEYS = ['__proto__', 'constructor', 'prototype'] as const;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Check if a key is dangerous for object property assignment.
|
|
12
|
+
* Handles both simple keys and dot-notation paths.
|
|
13
|
+
*/
|
|
14
|
+
export function isDangerousKey(key: string): boolean {
|
|
15
|
+
if (DANGEROUS_KEYS.includes(key as typeof DANGEROUS_KEYS[number])) {
|
|
16
|
+
return true;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
if (key.includes('.')) {
|
|
20
|
+
return key.split('.').some(part =>
|
|
21
|
+
DANGEROUS_KEYS.includes(part as typeof DANGEROUS_KEYS[number])
|
|
22
|
+
);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return false;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Filter out dangerous keys from an object.
|
|
30
|
+
* Returns a new object with only safe keys.
|
|
31
|
+
*/
|
|
32
|
+
export function sanitizeKeys<T extends Record<string, unknown>>(obj: T): T {
|
|
33
|
+
return Object.fromEntries(
|
|
34
|
+
Object.entries(obj).filter(([key]) => !isDangerousKey(key))
|
|
35
|
+
) as T;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Recursively sanitize an object, removing dangerous keys at all levels.
|
|
40
|
+
* Use this for deep merge operations.
|
|
41
|
+
*/
|
|
42
|
+
export function sanitizeDeep<T>(obj: T): T {
|
|
43
|
+
if (obj === null || typeof obj !== 'object') {
|
|
44
|
+
return obj;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (Array.isArray(obj)) {
|
|
48
|
+
return obj.map(item => sanitizeDeep(item)) as T;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const result: Record<string, unknown> = {};
|
|
52
|
+
|
|
53
|
+
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
|
|
54
|
+
if (!isDangerousKey(key)) {
|
|
55
|
+
result[key] = sanitizeDeep(value);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return result as T;
|
|
60
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { StringRecord } from '../types/common.types.js';
|
|
2
|
+
import { isValidS3KeySegment } from '../concerns/s3-key.js';
|
|
2
3
|
|
|
3
4
|
export interface PartitionFieldsDef {
|
|
4
5
|
[fieldName: string]: string;
|
|
@@ -62,6 +63,8 @@ export function validateResourceConfig(config: ResourceConfigInput): ValidationR
|
|
|
62
63
|
errors.push("Resource 'name' must be a string");
|
|
63
64
|
} else if (config.name.trim() === '') {
|
|
64
65
|
errors.push("Resource 'name' cannot be empty");
|
|
66
|
+
} else if (!isValidS3KeySegment(config.name)) {
|
|
67
|
+
errors.push(`Resource 'name' must be URL-friendly (no /, \\, =, or %). Got: '${config.name}'`);
|
|
65
68
|
}
|
|
66
69
|
|
|
67
70
|
if (!config.client) {
|
|
@@ -128,7 +131,9 @@ export function validateResourceConfig(config: ResourceConfigInput): ValidationR
|
|
|
128
131
|
errors.push("Resource 'partitions' must be an object");
|
|
129
132
|
} else {
|
|
130
133
|
for (const [partitionName, partitionDef] of Object.entries(config.partitions)) {
|
|
131
|
-
if (
|
|
134
|
+
if (!isValidS3KeySegment(partitionName)) {
|
|
135
|
+
errors.push(`Partition name '${partitionName}' must be URL-friendly (no /, \\, =, or %)`);
|
|
136
|
+
} else if (typeof partitionDef !== 'object' || Array.isArray(partitionDef)) {
|
|
132
137
|
errors.push(`Partition '${partitionName}' must be an object`);
|
|
133
138
|
} else if (!partitionDef.fields) {
|
|
134
139
|
errors.push(`Partition '${partitionName}' must have a 'fields' property`);
|
|
@@ -136,7 +141,9 @@ export function validateResourceConfig(config: ResourceConfigInput): ValidationR
|
|
|
136
141
|
errors.push(`Partition '${partitionName}.fields' must be an object`);
|
|
137
142
|
} else {
|
|
138
143
|
for (const [fieldName, fieldType] of Object.entries(partitionDef.fields)) {
|
|
139
|
-
if (
|
|
144
|
+
if (!isValidS3KeySegment(fieldName)) {
|
|
145
|
+
errors.push(`Partition field '${fieldName}' must be URL-friendly (no /, \\, =, or %)`);
|
|
146
|
+
} else if (typeof fieldType !== 'string') {
|
|
140
147
|
errors.push(`Partition '${partitionName}.fields.${fieldName}' must be a string`);
|
|
141
148
|
}
|
|
142
149
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { join } from 'path';
|
|
2
2
|
import { tryFn } from '../concerns/try-fn.js';
|
|
3
|
+
import { validateS3KeySegment } from '../concerns/s3-key.js';
|
|
3
4
|
import { PartitionError, ResourceError } from '../errors.js';
|
|
4
5
|
import type { StringRecord } from '../types/common.types.js';
|
|
5
6
|
|
|
@@ -361,7 +362,11 @@ export class ResourcePartitions {
|
|
|
361
362
|
}
|
|
362
363
|
|
|
363
364
|
extractValuesFromKey(id: string, keys: string[], sortedFields: Array<[string, string]>): StringRecord {
|
|
364
|
-
const
|
|
365
|
+
const idSegment = `id=${id}`;
|
|
366
|
+
const keyForId = keys.find(key => {
|
|
367
|
+
const segments = key.split('/');
|
|
368
|
+
return segments.some(segment => segment === idSegment);
|
|
369
|
+
});
|
|
365
370
|
if (!keyForId) {
|
|
366
371
|
throw new PartitionError(`Partition key not found for ID ${id}`, {
|
|
367
372
|
resourceName: this.resource.name,
|
|
@@ -547,6 +552,14 @@ export class ResourcePartitions {
|
|
|
547
552
|
}
|
|
548
553
|
|
|
549
554
|
async getFromPartition({ id, partitionName, partitionValues = {} }: GetFromPartitionParams): Promise<ResourceData> {
|
|
555
|
+
validateS3KeySegment(id, 'id');
|
|
556
|
+
|
|
557
|
+
for (const [fieldName, value] of Object.entries(partitionValues)) {
|
|
558
|
+
if (value !== undefined && value !== null) {
|
|
559
|
+
validateS3KeySegment(value, `partitionValues.${fieldName}`);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
|
|
550
563
|
const partitions = this.getPartitions();
|
|
551
564
|
if (!partitions || !partitions[partitionName]) {
|
|
552
565
|
throw new PartitionError(`Partition '${partitionName}' not found`, {
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { tryFn } from '../concerns/try-fn.js';
|
|
2
2
|
import { isEmpty, isObject } from 'lodash-es';
|
|
3
3
|
import { getBehavior } from '../behaviors/index.js';
|
|
4
|
+
import { isNotFoundError } from '../concerns/s3-errors.js';
|
|
5
|
+
import { sanitizeDeep } from '../concerns/safe-merge.js';
|
|
4
6
|
import { calculateTotalSize, calculateEffectiveLimit } from '../concerns/calculator.js';
|
|
5
7
|
import { mapAwsError, InvalidResourceItem, ResourceError, ValidationError } from '../errors.js';
|
|
6
8
|
import { streamToString } from '../stream/index.js';
|
|
@@ -240,11 +242,11 @@ export class ResourcePersistence {
|
|
|
240
242
|
}
|
|
241
243
|
|
|
242
244
|
const attributesWithDefaults = this.validator.applyDefaults(attributes);
|
|
243
|
-
const completeData: ResourceData = id !== undefined
|
|
245
|
+
const completeData: ResourceData = sanitizeDeep(id !== undefined
|
|
244
246
|
? { id, ...attributesWithDefaults }
|
|
245
|
-
: { ...attributesWithDefaults };
|
|
247
|
+
: { ...attributesWithDefaults }) as ResourceData;
|
|
246
248
|
|
|
247
|
-
const preProcessedData = await this.resource.executeHooks('beforeInsert', completeData) as ResourceData;
|
|
249
|
+
const preProcessedData = sanitizeDeep(await this.resource.executeHooks('beforeInsert', completeData)) as ResourceData;
|
|
248
250
|
|
|
249
251
|
const extraProps = Object.keys(preProcessedData).filter(
|
|
250
252
|
k => !(k in completeData) || preProcessedData[k] !== completeData[k]
|
|
@@ -501,7 +503,7 @@ export class ResourcePersistence {
|
|
|
501
503
|
async getOrNull(id: string): Promise<ResourceData | null> {
|
|
502
504
|
const [ok, err, data] = await tryFn<ResourceData>(() => this.get(id));
|
|
503
505
|
|
|
504
|
-
if (!ok && err && (
|
|
506
|
+
if (!ok && err && isNotFoundError(err)) {
|
|
505
507
|
return null;
|
|
506
508
|
}
|
|
507
509
|
|
|
@@ -512,7 +514,7 @@ export class ResourcePersistence {
|
|
|
512
514
|
async getOrThrow(id: string): Promise<ResourceData> {
|
|
513
515
|
const [ok, err, data] = await tryFn<ResourceData>(() => this.get(id));
|
|
514
516
|
|
|
515
|
-
if (!ok && err && (
|
|
517
|
+
if (!ok && err && isNotFoundError(err)) {
|
|
516
518
|
throw new ResourceError(`Resource '${this.name}' with id '${id}' not found`, {
|
|
517
519
|
resourceName: this.name,
|
|
518
520
|
operation: 'getOrThrow',
|
|
@@ -529,7 +531,13 @@ export class ResourcePersistence {
|
|
|
529
531
|
await this.resource.executeHooks('beforeExists', { id });
|
|
530
532
|
|
|
531
533
|
const key = this.resource.getResourceKey(id);
|
|
532
|
-
const [ok] = await tryFn(() => this.client.headObject(key));
|
|
534
|
+
const [ok, err] = await tryFn(() => this.client.headObject(key));
|
|
535
|
+
|
|
536
|
+
if (!ok && err) {
|
|
537
|
+
if (!isNotFoundError(err)) {
|
|
538
|
+
throw err;
|
|
539
|
+
}
|
|
540
|
+
}
|
|
533
541
|
|
|
534
542
|
await this.resource.executeHooks('afterExists', { id, exists: ok });
|
|
535
543
|
return ok;
|
|
@@ -728,7 +736,9 @@ export class ResourcePersistence {
|
|
|
728
736
|
(mergedData.metadata as StringRecord).updatedAt = now;
|
|
729
737
|
}
|
|
730
738
|
|
|
731
|
-
|
|
739
|
+
mergedData = sanitizeDeep(mergedData) as ResourceData;
|
|
740
|
+
|
|
741
|
+
const preProcessedData = sanitizeDeep(await this.resource.executeHooks('beforeUpdate', mergedData)) as ResourceData;
|
|
732
742
|
const completeData = { ...originalData, ...preProcessedData, id };
|
|
733
743
|
|
|
734
744
|
const { isValid, errors, data } = await this.resource.validate(completeData, { includeId: true });
|
|
@@ -794,10 +804,6 @@ export class ResourcePersistence {
|
|
|
794
804
|
if (okParse) finalContentType = 'application/json';
|
|
795
805
|
}
|
|
796
806
|
|
|
797
|
-
if (this.versioningEnabled && originalData._v !== this.version) {
|
|
798
|
-
await this.resource.createHistoricalVersion(id, originalData);
|
|
799
|
-
}
|
|
800
|
-
|
|
801
807
|
const [ok, err] = await tryFn(() => this.client.putObject({
|
|
802
808
|
key,
|
|
803
809
|
body: finalBody,
|
|
@@ -843,6 +849,18 @@ export class ResourcePersistence {
|
|
|
843
849
|
});
|
|
844
850
|
}
|
|
845
851
|
|
|
852
|
+
if (this.versioningEnabled && originalData._v !== this.version) {
|
|
853
|
+
const [okHistory, errHistory] = await tryFn(() => this.resource.createHistoricalVersion(id, originalData));
|
|
854
|
+
if (!okHistory) {
|
|
855
|
+
this.resource.emit('historyError', {
|
|
856
|
+
operation: 'update',
|
|
857
|
+
id,
|
|
858
|
+
error: errHistory,
|
|
859
|
+
message: (errHistory as Error).message
|
|
860
|
+
});
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
|
|
846
864
|
const updatedData = await this.resource.composeFullObjectFromWrite({
|
|
847
865
|
id,
|
|
848
866
|
metadata: finalMetadata,
|
|
@@ -985,6 +1003,8 @@ export class ResourcePersistence {
|
|
|
985
1003
|
mergedData.updatedAt = new Date().toISOString();
|
|
986
1004
|
}
|
|
987
1005
|
|
|
1006
|
+
mergedData = sanitizeDeep(mergedData) as ResourceData;
|
|
1007
|
+
|
|
988
1008
|
const { isValid, errors } = await this.validator.validate(mergedData);
|
|
989
1009
|
if (!isValid) {
|
|
990
1010
|
throw new ValidationError('Validation failed during patch', {
|
|
@@ -1057,7 +1077,7 @@ export class ResourcePersistence {
|
|
|
1057
1077
|
attributesWithDefaults.updatedAt = new Date().toISOString();
|
|
1058
1078
|
}
|
|
1059
1079
|
|
|
1060
|
-
const completeData: ResourceData = { id, ...attributesWithDefaults };
|
|
1080
|
+
const completeData: ResourceData = sanitizeDeep({ id, ...attributesWithDefaults }) as ResourceData;
|
|
1061
1081
|
|
|
1062
1082
|
const {
|
|
1063
1083
|
errors,
|
|
@@ -1230,7 +1250,9 @@ export class ResourcePersistence {
|
|
|
1230
1250
|
(mergedData.metadata as StringRecord).updatedAt = now;
|
|
1231
1251
|
}
|
|
1232
1252
|
|
|
1233
|
-
|
|
1253
|
+
mergedData = sanitizeDeep(mergedData) as ResourceData;
|
|
1254
|
+
|
|
1255
|
+
const preProcessedData = sanitizeDeep(await this.resource.executeHooks('beforeUpdate', mergedData)) as ResourceData;
|
|
1234
1256
|
const completeData = { ...originalData, ...preProcessedData, id };
|
|
1235
1257
|
|
|
1236
1258
|
const { isValid, errors, data } = await this.resource.validate(completeData, { includeId: true });
|
|
@@ -1301,6 +1323,18 @@ export class ResourcePersistence {
|
|
|
1301
1323
|
};
|
|
1302
1324
|
}
|
|
1303
1325
|
|
|
1326
|
+
if (this.versioningEnabled && originalData._v !== this.version) {
|
|
1327
|
+
const [okHistory, errHistory] = await tryFn(() => this.resource.createHistoricalVersion(id, originalData));
|
|
1328
|
+
if (!okHistory) {
|
|
1329
|
+
this.resource.emit('historyError', {
|
|
1330
|
+
operation: 'updateConditional',
|
|
1331
|
+
id,
|
|
1332
|
+
error: errHistory,
|
|
1333
|
+
message: (errHistory as Error).message
|
|
1334
|
+
});
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1304
1338
|
const updatedData = await this.resource.composeFullObjectFromWrite({
|
|
1305
1339
|
id,
|
|
1306
1340
|
metadata: processedMetadata,
|
|
@@ -400,60 +400,35 @@ export class ResourceQuery {
|
|
|
400
400
|
}
|
|
401
401
|
|
|
402
402
|
async page({ offset = 0, size = 100, partition = null, partitionValues = {}, skipCount = false }: PageParams = {}): Promise<PageResult> {
|
|
403
|
-
const
|
|
404
|
-
let totalItems: number | null = null;
|
|
405
|
-
let totalPages: number | null = null;
|
|
406
|
-
if (!skipCount) {
|
|
407
|
-
const [okCount, , count] = await tryFn<number>(() => this.count({ partition, partitionValues }));
|
|
408
|
-
if (okCount && count !== undefined) {
|
|
409
|
-
totalItems = count;
|
|
410
|
-
totalPages = Math.ceil(totalItems / size);
|
|
411
|
-
}
|
|
412
|
-
}
|
|
403
|
+
const effectiveSize = size > 0 ? size : 100;
|
|
413
404
|
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
405
|
+
let totalItems: number | null = null;
|
|
406
|
+
let totalPages: number | null = null;
|
|
407
|
+
if (!skipCount) {
|
|
408
|
+
totalItems = await this.count({ partition, partitionValues });
|
|
409
|
+
totalPages = Math.ceil(totalItems / effectiveSize);
|
|
410
|
+
}
|
|
420
411
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
totalItems,
|
|
424
|
-
page,
|
|
425
|
-
pageSize: size,
|
|
426
|
-
totalPages,
|
|
427
|
-
hasMore: items.length === size && (offset + size) < (totalItems || Infinity),
|
|
428
|
-
_debug: {
|
|
429
|
-
requestedSize: size,
|
|
430
|
-
requestedOffset: offset,
|
|
431
|
-
actualItemsReturned: items.length,
|
|
432
|
-
skipCount,
|
|
433
|
-
hasTotalItems: totalItems !== null
|
|
434
|
-
}
|
|
435
|
-
};
|
|
436
|
-
this.resource._emitStandardized('paginated', pageResult);
|
|
437
|
-
return pageResult;
|
|
438
|
-
});
|
|
412
|
+
const page = Math.floor(offset / effectiveSize);
|
|
413
|
+
const items = await this.list({ partition, partitionValues, limit: effectiveSize, offset });
|
|
439
414
|
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
hasMore: false,
|
|
415
|
+
const pageResult: PageResult = {
|
|
416
|
+
items,
|
|
417
|
+
totalItems,
|
|
418
|
+
page,
|
|
419
|
+
pageSize: effectiveSize,
|
|
420
|
+
totalPages,
|
|
421
|
+
hasMore: items.length === effectiveSize && (offset + effectiveSize) < (totalItems || Infinity),
|
|
448
422
|
_debug: {
|
|
449
423
|
requestedSize: size,
|
|
450
424
|
requestedOffset: offset,
|
|
451
|
-
actualItemsReturned:
|
|
452
|
-
skipCount
|
|
453
|
-
hasTotalItems:
|
|
454
|
-
error: (err as Error).message
|
|
425
|
+
actualItemsReturned: items.length,
|
|
426
|
+
skipCount,
|
|
427
|
+
hasTotalItems: totalItems !== null
|
|
455
428
|
}
|
|
456
429
|
};
|
|
430
|
+
this.resource._emitStandardized('paginated', pageResult);
|
|
431
|
+
return pageResult;
|
|
457
432
|
}
|
|
458
433
|
|
|
459
434
|
async query(filter: StringRecord = {}, { limit = 100, offset = 0, partition = null, partitionValues = {} }: QueryOptions = {}): Promise<ResourceData[]> {
|