prostgles-server 4.0.16 → 4.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AuthHandler.js +214 -210
- package/dist/AuthHandler.js.map +1 -1
- package/dist/DBEventsManager.js +34 -31
- package/dist/DBEventsManager.js.map +1 -1
- package/dist/DBSchemaBuilder.js +3 -2
- package/dist/DBSchemaBuilder.js.map +1 -1
- package/dist/DboBuilder/QueryBuilder/QueryBuilder.d.ts +19 -14
- package/dist/DboBuilder/QueryBuilder/QueryBuilder.d.ts.map +1 -1
- package/dist/DboBuilder/QueryBuilder/QueryBuilder.js +159 -151
- package/dist/DboBuilder/QueryBuilder/QueryBuilder.js.map +1 -1
- package/dist/DboBuilder/TableHandler.d.ts +9 -14
- package/dist/DboBuilder/TableHandler.d.ts.map +1 -1
- package/dist/DboBuilder/TableHandler.js +21 -41
- package/dist/DboBuilder/TableHandler.js.map +1 -1
- package/dist/DboBuilder/ViewHandler.d.ts +3 -3
- package/dist/DboBuilder/ViewHandler.d.ts.map +1 -1
- package/dist/DboBuilder/ViewHandler.js +24 -9
- package/dist/DboBuilder/ViewHandler.js.map +1 -1
- package/dist/DboBuilder/getSubscribeRelatedTables.d.ts +1 -1
- package/dist/DboBuilder/getSubscribeRelatedTables.d.ts.map +1 -1
- package/dist/DboBuilder/parseUpdateRules.d.ts.map +1 -1
- package/dist/DboBuilder/parseUpdateRules.js.map +1 -1
- package/dist/DboBuilder/runSQL.js +5 -5
- package/dist/DboBuilder/runSQL.js.map +1 -1
- package/dist/DboBuilder.d.ts.map +1 -1
- package/dist/DboBuilder.js +85 -78
- package/dist/DboBuilder.js.map +1 -1
- package/dist/FileManager/FileManager.d.ts +143 -0
- package/dist/FileManager/FileManager.d.ts.map +1 -0
- package/dist/FileManager/FileManager.js +278 -0
- package/dist/FileManager/FileManager.js.map +1 -0
- package/dist/FileManager/initFileManager.d.ts +4 -0
- package/dist/FileManager/initFileManager.d.ts.map +1 -0
- package/dist/FileManager/initFileManager.js +209 -0
- package/dist/FileManager/initFileManager.js.map +1 -0
- package/dist/FileManager/parseFile.d.ts +15 -0
- package/dist/FileManager/parseFile.d.ts.map +1 -0
- package/dist/FileManager/parseFile.js +59 -0
- package/dist/FileManager/parseFile.js.map +1 -0
- package/dist/FileManager/upload.d.ts +6 -0
- package/dist/FileManager/upload.d.ts.map +1 -0
- package/dist/FileManager/upload.js +76 -0
- package/dist/FileManager/upload.js.map +1 -0
- package/dist/FileManager/uploadStream.d.ts +5 -0
- package/dist/FileManager/uploadStream.d.ts.map +1 -0
- package/dist/FileManager/uploadStream.js +70 -0
- package/dist/FileManager/uploadStream.js.map +1 -0
- package/dist/Filtering.d.ts +1 -1
- package/dist/Filtering.d.ts.map +1 -1
- package/dist/PostgresNotifListenManager.js +26 -20
- package/dist/PostgresNotifListenManager.js.map +1 -1
- package/dist/Prostgles.d.ts +2 -2
- package/dist/Prostgles.d.ts.map +1 -1
- package/dist/Prostgles.js +192 -172
- package/dist/Prostgles.js.map +1 -1
- package/dist/PubSubManager/PubSubManager.d.ts +3 -3
- package/dist/PubSubManager/PubSubManager.js +164 -154
- package/dist/PubSubManager/PubSubManager.js.map +1 -1
- package/dist/PublishParser.d.ts +13 -13
- package/dist/PublishParser.d.ts.map +1 -1
- package/dist/PublishParser.js +8 -2
- package/dist/PublishParser.js.map +1 -1
- package/dist/SchemaWatch.js +2 -1
- package/dist/SchemaWatch.js.map +1 -1
- package/dist/TableConfig/TableConfig.d.ts.map +1 -1
- package/dist/TableConfig/TableConfig.js +100 -92
- package/dist/TableConfig/TableConfig.js.map +1 -1
- package/lib/AuthHandler.js +1 -1
- package/lib/AuthHandler.ts +1 -1
- package/lib/DBSchemaBuilder.js +3 -2
- package/lib/DBSchemaBuilder.ts +3 -2
- package/lib/DboBuilder/TableHandler.d.ts +4 -9
- package/lib/DboBuilder/TableHandler.d.ts.map +1 -1
- package/lib/DboBuilder/TableHandler.js +1 -22
- package/lib/DboBuilder/TableHandler.ts +3 -31
- package/lib/DboBuilder/ViewHandler.ts +1 -1
- package/lib/DboBuilder/getSubscribeRelatedTables.d.ts +1 -1
- package/lib/DboBuilder/getSubscribeRelatedTables.d.ts.map +1 -1
- package/lib/DboBuilder/getSubscribeRelatedTables.ts +1 -1
- package/lib/DboBuilder/parseUpdateRules.d.ts.map +1 -1
- package/lib/DboBuilder/parseUpdateRules.ts +2 -2
- package/lib/DboBuilder.d.ts.map +1 -1
- package/lib/DboBuilder.js +0 -13
- package/lib/DboBuilder.ts +1 -16
- package/lib/{FileManager.d.ts → FileManager/FileManager.d.ts} +12 -20
- package/lib/FileManager/FileManager.d.ts.map +1 -0
- package/lib/FileManager/FileManager.js +303 -0
- package/lib/FileManager/FileManager.ts +378 -0
- package/lib/FileManager/initFileManager.d.ts +4 -0
- package/lib/FileManager/initFileManager.d.ts.map +1 -0
- package/lib/FileManager/initFileManager.js +231 -0
- package/lib/FileManager/initFileManager.ts +236 -0
- package/lib/FileManager/parseFile.d.ts +15 -0
- package/lib/FileManager/parseFile.d.ts.map +1 -0
- package/lib/FileManager/parseFile.js +58 -0
- package/lib/FileManager/parseFile.ts +69 -0
- package/lib/FileManager/upload.d.ts +6 -0
- package/lib/FileManager/upload.d.ts.map +1 -0
- package/lib/FileManager/upload.js +98 -0
- package/lib/FileManager/upload.ts +83 -0
- package/lib/FileManager/uploadStream.d.ts +5 -0
- package/lib/FileManager/uploadStream.d.ts.map +1 -0
- package/lib/FileManager/uploadStream.js +92 -0
- package/lib/FileManager/uploadStream.ts +80 -0
- package/lib/Filtering.d.ts +1 -1
- package/lib/Filtering.d.ts.map +1 -1
- package/lib/Filtering.ts +1 -1
- package/lib/JSONBValidation/validate_jsonb_schema_sql.ts +1 -1
- package/lib/Prostgles.d.ts +2 -2
- package/lib/Prostgles.d.ts.map +1 -1
- package/lib/Prostgles.js +8 -5
- package/lib/Prostgles.ts +10 -6
- package/lib/PublishParser.d.ts +13 -13
- package/lib/PublishParser.d.ts.map +1 -1
- package/lib/PublishParser.js +1 -0
- package/lib/PublishParser.ts +13 -12
- package/lib/TableConfig/TableConfig.d.ts.map +1 -1
- package/lib/TableConfig/TableConfig.js +12 -7
- package/lib/TableConfig/TableConfig.ts +13 -8
- package/package.json +3 -3
- package/tests/client/PID.txt +1 -1
- package/tests/client/index.d.ts.map +1 -1
- package/tests/client/index.ts +1 -3
- package/tests/client/package-lock.json +31 -4561
- package/tests/client/package.json +2 -2
- package/tests/client/tsconfig.json +1 -0
- package/tests/isomorphic_queries.d.ts.map +1 -1
- package/tests/isomorphic_queries.ts +3 -3
- package/tests/server/package-lock.json +13 -13
- package/tests/server/package.json +1 -1
- package/tests/server/tsconfig.json +1 -0
- package/tsconfig.json +3 -2
- package/lib/FileManager.d.ts.map +0 -1
- package/lib/FileManager.js +0 -676
- package/lib/FileManager.ts +0 -811
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
|
|
2
|
+
import { S3, IAM } from 'aws-sdk';
|
|
3
|
+
// import { PutObjectCommand, S3 } from "@aws-sdk/client-s3";
|
|
4
|
+
import * as fs from 'fs';
|
|
5
|
+
import * as stream from 'stream';
|
|
6
|
+
|
|
7
|
+
import * as sharp from "sharp";
|
|
8
|
+
import checkDiskSpace from 'check-disk-space';
|
|
9
|
+
|
|
10
|
+
import { DB, DBHandlerServer, ExpressApp, Prostgles } from '../Prostgles';
|
|
11
|
+
import { ALLOWED_CONTENT_TYPE, ALLOWED_EXTENSION, CONTENT_TYPE_TO_EXT, getKeys, isDefined, ValidatedColumnInfo } from 'prostgles-types';
|
|
12
|
+
|
|
13
|
+
import AWS from 'aws-sdk';
|
|
14
|
+
import * as path from "path";
|
|
15
|
+
import { initFileManager } from "./initFileManager";
|
|
16
|
+
import { parseFile } from "./parseFile";
|
|
17
|
+
import { upload } from "./upload";
|
|
18
|
+
import { uploadStream } from "./uploadStream";
|
|
19
|
+
|
|
20
|
+
export const HOUR = 3600 * 1000;
|
|
21
|
+
|
|
22
|
+
export const asSQLIdentifier = async (name: string, db: DB): Promise<string> => {
|
|
23
|
+
return (await db.one("select format('%I', $1) as name", [name]))?.name
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type OnProgress = (progress: S3.ManagedUpload.Progress) => void
|
|
27
|
+
|
|
28
|
+
export type ImageOptions = {
|
|
29
|
+
keepMetadata?: boolean;
|
|
30
|
+
compression?:
|
|
31
|
+
/**
|
|
32
|
+
* Will resize image maintaing scale ratio
|
|
33
|
+
*/
|
|
34
|
+
| { inside: { width: number; height: number } }
|
|
35
|
+
| { contain:
|
|
36
|
+
| { width: number }
|
|
37
|
+
| { height: number }
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type S3Config = {
|
|
42
|
+
region: string;
|
|
43
|
+
bucket: string;
|
|
44
|
+
accessKeyId: string;
|
|
45
|
+
secretAccessKey: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export type LocalConfig = {
|
|
49
|
+
/**
|
|
50
|
+
* example: path.join(__dirname+'/media')
|
|
51
|
+
* note that this location will be relative to the compiled file location
|
|
52
|
+
*/
|
|
53
|
+
localFolderPath: string;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Minimum amount of free bytes available to allow saving files
|
|
57
|
+
* Defaults to 100MB
|
|
58
|
+
*/
|
|
59
|
+
minFreeBytes?: number;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export type UploadItem = {
|
|
63
|
+
name: string;
|
|
64
|
+
content_type: string;
|
|
65
|
+
data: Buffer;
|
|
66
|
+
extension: string;
|
|
67
|
+
};
|
|
68
|
+
export type UploadedItem = {
|
|
69
|
+
/**
|
|
70
|
+
* Url that is passed to client
|
|
71
|
+
*/
|
|
72
|
+
url: string;
|
|
73
|
+
filePath?: string;
|
|
74
|
+
etag: string;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* S3 url of the resource
|
|
78
|
+
*/
|
|
79
|
+
s3_url?: string;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Total uploaded file size in bytes
|
|
83
|
+
*/
|
|
84
|
+
content_length: number;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
export class FileManager {
|
|
88
|
+
|
|
89
|
+
static testCredentials = async (accessKeyId: string, secretAccessKey: string) => {
|
|
90
|
+
const sts = new AWS.STS();
|
|
91
|
+
AWS.config.credentials = {
|
|
92
|
+
accessKeyId,
|
|
93
|
+
secretAccessKey
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const ident = await sts.getCallerIdentity({}).promise();
|
|
97
|
+
return ident;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
s3Client?: S3;
|
|
101
|
+
|
|
102
|
+
config: S3Config | LocalConfig;
|
|
103
|
+
imageOptions?: ImageOptions;
|
|
104
|
+
|
|
105
|
+
prostgles?: Prostgles;
|
|
106
|
+
get dbo(): DBHandlerServer {
|
|
107
|
+
if(!this.prostgles?.dbo) {
|
|
108
|
+
// this.prostgles?.refreshDBO();
|
|
109
|
+
throw "this.prostgles.dbo missing"
|
|
110
|
+
}
|
|
111
|
+
return this.prostgles.dbo
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
get db(): DB {
|
|
115
|
+
if(!this.prostgles?.db) throw "this.prostgles.db missing"
|
|
116
|
+
return this.prostgles.db
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
tableName?: string;
|
|
120
|
+
|
|
121
|
+
fileRoute?: string;
|
|
122
|
+
get fileRouteExpress (){
|
|
123
|
+
return this.fileRoute + "/:name";
|
|
124
|
+
}
|
|
125
|
+
private checkInterval?: NodeJS.Timeout;
|
|
126
|
+
|
|
127
|
+
constructor(config: FileManager["config"], imageOptions?: ImageOptions){
|
|
128
|
+
this.config = config;
|
|
129
|
+
this.imageOptions = imageOptions;
|
|
130
|
+
|
|
131
|
+
if("region" in config){
|
|
132
|
+
const { region, accessKeyId, secretAccessKey } = config;
|
|
133
|
+
this.s3Client = new S3({
|
|
134
|
+
region,
|
|
135
|
+
credentials: { accessKeyId, secretAccessKey },
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const fullConfig = this.prostgles?.opts.fileTable;
|
|
140
|
+
if(fullConfig?.delayedDelete){
|
|
141
|
+
this.checkInterval = setInterval(async () => {
|
|
142
|
+
const fileTable = fullConfig.tableName;
|
|
143
|
+
const daysDelay = fullConfig.delayedDelete?.deleteAfterNDays ?? 0;
|
|
144
|
+
if(fileTable && this.dbo[fileTable]?.delete && daysDelay){
|
|
145
|
+
const filesToDelete = await this.dbo[fileTable]?.find?.({ deleted_from_storage: null, deleted: { ">": Date.now() - (daysDelay * HOUR * 24) } }) ?? [];
|
|
146
|
+
for await(const file of filesToDelete){
|
|
147
|
+
await this.deleteFile(file.name)
|
|
148
|
+
}
|
|
149
|
+
} else {
|
|
150
|
+
console.error("FileManager checkInterval delayedDelete FAIL: Could not access file table tableHandler.delete()")
|
|
151
|
+
}
|
|
152
|
+
}, Math.max(10000, (fullConfig.delayedDelete.checkIntervalHours || 0) * HOUR))
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async getFileStream(name: string): Promise<stream.Readable> {
|
|
157
|
+
if("bucket" in this.config && this.s3Client){
|
|
158
|
+
return this.s3Client.getObject({ Key: name, Bucket: this.config.bucket }).createReadStream()
|
|
159
|
+
} else if("localFolderPath" in this.config){
|
|
160
|
+
const filePath = path.resolve(`${this.config.localFolderPath}/${name}`);
|
|
161
|
+
if(!fs.existsSync(filePath)){
|
|
162
|
+
throw `File ${filePath} could not be found`;
|
|
163
|
+
}
|
|
164
|
+
return fs.createReadStream(filePath, { encoding: undefined })
|
|
165
|
+
} else throw new Error("Not expected");
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async deleteFile(name: string) {
|
|
169
|
+
if("bucket" in this.config && this.s3Client){
|
|
170
|
+
const res = await this.s3Client?.deleteObject({ Bucket: this.config.bucket, Key: name }).promise();
|
|
171
|
+
return res;
|
|
172
|
+
} else if("localFolderPath" in this.config){
|
|
173
|
+
const path = `${this.config.localFolderPath}/${name}`;
|
|
174
|
+
if(!fs.existsSync(path)){
|
|
175
|
+
throw `File ${path} could not be found`;
|
|
176
|
+
}
|
|
177
|
+
fs.unlinkSync(path);
|
|
178
|
+
if(fs.existsSync(path)) throw new Error("Could not delete file");
|
|
179
|
+
}
|
|
180
|
+
return true
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
parseFile = parseFile.bind(this);
|
|
184
|
+
|
|
185
|
+
getLocalFileUrl = (name: string) => this.fileRoute? `${this.fileRoute}/${name}` : "";
|
|
186
|
+
|
|
187
|
+
checkFreeSpace = async (folderPath: string, fileSize = 0) => {
|
|
188
|
+
if(!this.s3Client && "localFolderPath" in this.config) {
|
|
189
|
+
const { minFreeBytes = 1.048e6 } = this.config;
|
|
190
|
+
const required = Math.max(fileSize, minFreeBytes)
|
|
191
|
+
if(required){
|
|
192
|
+
const diskSpace = await checkDiskSpace(folderPath);
|
|
193
|
+
if(diskSpace.free < required){
|
|
194
|
+
const err = `There is not enough space on the server to save files.\nTotal: ${bytesToSize(diskSpace.size)} \nRemaning: ${bytesToSize(diskSpace.free)} \nRequired: ${bytesToSize(required)}`
|
|
195
|
+
throw new Error(err);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
uploadStream = uploadStream.bind(this);
|
|
202
|
+
|
|
203
|
+
upload = upload.bind(this);
|
|
204
|
+
|
|
205
|
+
uploadAsMedia = async (params: {
|
|
206
|
+
item: UploadItem;
|
|
207
|
+
allowedExtensions?: Array<ALLOWED_EXTENSION>;
|
|
208
|
+
dissallowedExtensions?: Array<ALLOWED_EXTENSION>;
|
|
209
|
+
imageOptions?: ImageOptions;
|
|
210
|
+
}): Promise<UploadedItem> => {
|
|
211
|
+
const { item, imageOptions } = params;
|
|
212
|
+
const { name, data, content_type, extension } = item;
|
|
213
|
+
if(!data) throw "No file provided";
|
|
214
|
+
if(!name || typeof name !== "string") throw "Expecting a string name";
|
|
215
|
+
|
|
216
|
+
// const type = await this.getMIME(data, name, allowedExtensions, dissallowedExtensions);
|
|
217
|
+
|
|
218
|
+
let _data = data;
|
|
219
|
+
|
|
220
|
+
/** Resize/compress/remove exif from photos */
|
|
221
|
+
if(content_type.startsWith("image") && extension.toLowerCase() !== "gif"){
|
|
222
|
+
|
|
223
|
+
const compression = imageOptions?.compression
|
|
224
|
+
if(compression){
|
|
225
|
+
console.log("Resizing image")
|
|
226
|
+
let opts;
|
|
227
|
+
if("contain" in compression){
|
|
228
|
+
opts = {
|
|
229
|
+
fit: sharp.fit.contain,
|
|
230
|
+
...compression.contain
|
|
231
|
+
}
|
|
232
|
+
} else if("inside" in compression){
|
|
233
|
+
opts = {
|
|
234
|
+
fit: sharp.fit.inside,
|
|
235
|
+
...compression.inside
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
_data = await sharp(data)
|
|
239
|
+
.resize(opts as any)
|
|
240
|
+
.withMetadata(Boolean(imageOptions?.keepMetadata) as any)
|
|
241
|
+
// .jpeg({ quality: 80 })
|
|
242
|
+
.toBuffer()
|
|
243
|
+
} else if(!imageOptions?.keepMetadata) {
|
|
244
|
+
/**
|
|
245
|
+
* Remove exif
|
|
246
|
+
*/
|
|
247
|
+
// const metadata = await simg.metadata();
|
|
248
|
+
// const simg = await sharp(data);
|
|
249
|
+
|
|
250
|
+
_data = await sharp(data).clone().withMetadata({
|
|
251
|
+
exif: {}
|
|
252
|
+
})
|
|
253
|
+
.toBuffer()
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const res = await this.upload(_data, name, content_type);
|
|
258
|
+
|
|
259
|
+
return res
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async getFileS3URL(fileName: string, expiresInSeconds: number = 30 * 60){
|
|
263
|
+
const params = {
|
|
264
|
+
Bucket: (this.config as S3Config).bucket,
|
|
265
|
+
Key: fileName,
|
|
266
|
+
Expires: Math.round(expiresInSeconds || 30 * 60) // Error if float
|
|
267
|
+
};
|
|
268
|
+
return await this.s3Client?.getSignedUrlPromise("getObject", params);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
parseSQLIdentifier = async (name: string ) => asSQLIdentifier(name, this.prostgles!.db!);// this.prostgles.dbo.sql<"value">("select format('%I', $1)", [name], { returnType: "value" } )
|
|
272
|
+
|
|
273
|
+
getColInfo = (args: { tableName: string; colName: string }): ValidatedColumnInfo["file"] | undefined => {
|
|
274
|
+
const { colName, tableName } = args;
|
|
275
|
+
const tableConfig = this.prostgles?.opts.fileTable?.referencedTables?.[tableName];
|
|
276
|
+
const isReferencingFileTable = this.dbo[tableName]?.columns?.some(c => c.name === colName && c.references && c.references?.some(({ ftable }) => ftable === this.tableName ));
|
|
277
|
+
if(isReferencingFileTable){
|
|
278
|
+
if(tableConfig && typeof tableConfig !== "string"){
|
|
279
|
+
return tableConfig.referenceColumns[colName];
|
|
280
|
+
}
|
|
281
|
+
return { acceptedContent: "*" };
|
|
282
|
+
}
|
|
283
|
+
return undefined;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
init = initFileManager.bind(this);
|
|
287
|
+
|
|
288
|
+
destroy = () => {
|
|
289
|
+
removeExpressRoute(this.prostgles?.opts.fileTable?.expressApp, [this.fileRouteExpress]);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export const removeExpressRoute = (app: ExpressApp | undefined, routePaths: (string | undefined)[]) => {
|
|
294
|
+
|
|
295
|
+
const routes = app?._router?.stack;
|
|
296
|
+
if(routes){
|
|
297
|
+
routes.forEach((route, i) => {
|
|
298
|
+
if(routePaths.filter(isDefined).includes(route.route?.path as any)){
|
|
299
|
+
routes.splice(i, 1);
|
|
300
|
+
}
|
|
301
|
+
})
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export const getFileTypeFromFilename = (fileName: string): { mime: ALLOWED_CONTENT_TYPE; ext: ALLOWED_EXTENSION | string } | undefined => {
|
|
306
|
+
|
|
307
|
+
const nameParts = fileName.split(".");
|
|
308
|
+
|
|
309
|
+
if(nameParts.length < 2) return undefined;
|
|
310
|
+
|
|
311
|
+
const nameExt = nameParts.at(-1)!.toLowerCase(),
|
|
312
|
+
mime = getKeys(CONTENT_TYPE_TO_EXT).find(k => (CONTENT_TYPE_TO_EXT[k] as readonly string[]).includes(nameExt));
|
|
313
|
+
|
|
314
|
+
if(!mime) return undefined;
|
|
315
|
+
|
|
316
|
+
return {
|
|
317
|
+
mime,
|
|
318
|
+
ext: nameExt as ALLOWED_EXTENSION,
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// const fileType = require("file-type");
|
|
323
|
+
// const res = await fileType.fromBuffer(typeof file === "string"? Buffer.from(file, 'utf8') : file);
|
|
324
|
+
|
|
325
|
+
export const getFileType = async (file: Buffer | string, fileName: string): Promise<{ mime: ALLOWED_CONTENT_TYPE; ext: ALLOWED_EXTENSION }> => {
|
|
326
|
+
|
|
327
|
+
const { fileTypeFromBuffer } = await (eval('import("file-type")') as Promise<typeof import('file-type')>);
|
|
328
|
+
|
|
329
|
+
const fileNameMime = getFileTypeFromFilename(fileName);
|
|
330
|
+
if(!fileNameMime?.ext) throw new Error("File name must contain extenion")
|
|
331
|
+
const res = await fileTypeFromBuffer(typeof file === "string"? Buffer.from(file, 'utf8') : file);
|
|
332
|
+
|
|
333
|
+
if(!res) {
|
|
334
|
+
|
|
335
|
+
/* Set correct/missing extension */
|
|
336
|
+
const nameExt = fileNameMime?.ext;
|
|
337
|
+
if(["xml", "txt", "csv", "tsv", "svg", "sql"].includes(nameExt) && fileNameMime.mime){
|
|
338
|
+
return fileNameMime as any;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
throw new Error("Could not get the file type from file buffer");
|
|
342
|
+
} else {
|
|
343
|
+
|
|
344
|
+
if(!res.ext || fileNameMime?.ext.toLowerCase() !== res.ext.toLowerCase()){
|
|
345
|
+
throw new Error(`There is a mismatch between file name extension and actual buffer extension: ${fileNameMime?.ext } vs ${res.ext}`)
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return res as any;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
export function bytesToSize(bytes: number) {
|
|
352
|
+
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
|
353
|
+
if (bytes == 0) return '0 Byte';
|
|
354
|
+
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)) + "");
|
|
355
|
+
return (bytes / Math.pow(1024, i)).toFixed(1) + ' ' + sizes[i];
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
*
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
// if(content_type && typeof content_type !== "string") throw "Invalid content_type provided";
|
|
364
|
+
// if(title && typeof title !== "string") throw "Invalid title provided";
|
|
365
|
+
// let fExt = name.split(".").pop()
|
|
366
|
+
// if(content_type && name.split(".").length > 1 && fExt && fExt.length <= 4){
|
|
367
|
+
// type = {
|
|
368
|
+
// mime: content_type,
|
|
369
|
+
// ext: fExt,
|
|
370
|
+
// fileName: name,
|
|
371
|
+
// }
|
|
372
|
+
// } else {
|
|
373
|
+
// type = await this.getMIME(data, name);//, ["png", "jpg", "ogg", "webm", "pdf", "doc"]);
|
|
374
|
+
// }
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
*/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"initFileManager.d.ts","sourceRoot":"","sources":["initFileManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAqB,MAAM,eAAe,CAAC;AAI/D,wBAAsB,eAAe,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,iBAmOtE"}
|
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
+
exports.initFileManager = void 0;
|
|
27
|
+
const prostgles_types_1 = require("prostgles-types");
|
|
28
|
+
const runSQL_1 = require("../DboBuilder/runSQL");
|
|
29
|
+
const FileManager_1 = require("./FileManager");
|
|
30
|
+
const fs = __importStar(require("fs"));
|
|
31
|
+
async function initFileManager(prg) {
|
|
32
|
+
this.prostgles = prg;
|
|
33
|
+
// const { dbo, db, opts } = prg;
|
|
34
|
+
const { fileTable } = prg.opts;
|
|
35
|
+
if (!fileTable)
|
|
36
|
+
throw "fileTable missing";
|
|
37
|
+
const { tableName = "media", referencedTables = {} } = fileTable;
|
|
38
|
+
this.tableName = tableName;
|
|
39
|
+
const maxBfSizeMB = (prg.opts.io?.engine?.opts?.maxHttpBufferSize || 1e6) / 1e6;
|
|
40
|
+
console.log(`Prostgles: Initiated file manager. Max allowed file size: ${maxBfSizeMB}MB (maxHttpBufferSize = 1e6). To increase this set maxHttpBufferSize in socket.io server init options`);
|
|
41
|
+
// throw `this.db.tx(d => do everything in a transaction pls!!!!`;
|
|
42
|
+
const canCreate = await (0, runSQL_1.canCreateTables)(this.db);
|
|
43
|
+
const runQuery = (q) => {
|
|
44
|
+
if (!canCreate)
|
|
45
|
+
throw "File table creation failed. Your postgres user does not have CREATE table privileges";
|
|
46
|
+
return this.db.any(`
|
|
47
|
+
${q}
|
|
48
|
+
`);
|
|
49
|
+
/* ${PubSubManager.EXCLUDE_QUERY_FROM_SCHEMA_WATCH_ID} */
|
|
50
|
+
};
|
|
51
|
+
/**
|
|
52
|
+
* 1. Create media table
|
|
53
|
+
*/
|
|
54
|
+
if (!this.dbo[tableName]) {
|
|
55
|
+
console.log(`Creating fileTable ${(0, prostgles_types_1.asName)(tableName)} ...`);
|
|
56
|
+
await runQuery(`CREATE EXTENSION IF NOT EXISTS pgcrypto `);
|
|
57
|
+
await runQuery(`CREATE TABLE IF NOT EXISTS ${(0, prostgles_types_1.asName)(tableName)} (
|
|
58
|
+
name TEXT NOT NULL,
|
|
59
|
+
extension TEXT NOT NULL,
|
|
60
|
+
content_type TEXT NOT NULL,
|
|
61
|
+
content_length BIGINT NOT NULL DEFAULT 0,
|
|
62
|
+
added TIMESTAMP NOT NULL DEFAULT NOW(),
|
|
63
|
+
url TEXT NOT NULL,
|
|
64
|
+
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
65
|
+
original_name TEXT NOT NULL,
|
|
66
|
+
description TEXT,
|
|
67
|
+
s3_url TEXT,
|
|
68
|
+
signed_url TEXT,
|
|
69
|
+
signed_url_expires BIGINT,
|
|
70
|
+
etag TEXT,
|
|
71
|
+
deleted BIGINT,
|
|
72
|
+
deleted_from_storage BIGINT,
|
|
73
|
+
UNIQUE(id),
|
|
74
|
+
UNIQUE(name)
|
|
75
|
+
)`);
|
|
76
|
+
console.log(`Created fileTable ${(0, prostgles_types_1.asName)(tableName)}`);
|
|
77
|
+
await prg.refreshDBO();
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* 2. Create media lookup tables
|
|
81
|
+
*/
|
|
82
|
+
for (const refTable in referencedTables) {
|
|
83
|
+
if (!this.dbo[refTable]) {
|
|
84
|
+
throw `Referenced table (${refTable}) from fileTable.referencedTables prostgles init config does not exist`;
|
|
85
|
+
}
|
|
86
|
+
const cols = await this.dbo[refTable].getColumns();
|
|
87
|
+
const tableConfig = referencedTables[refTable];
|
|
88
|
+
if (typeof tableConfig !== "string") {
|
|
89
|
+
for (const colName of (0, prostgles_types_1.getKeys)(tableConfig.referenceColumns)) {
|
|
90
|
+
const existingCol = cols.find(c => c.name === colName);
|
|
91
|
+
if (existingCol) {
|
|
92
|
+
if (existingCol.references?.some(({ ftable }) => ftable === tableName)) {
|
|
93
|
+
// All ok
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
if (existingCol.udt_name === "uuid") {
|
|
97
|
+
try {
|
|
98
|
+
const query = `ALTER TABLE ${(0, prostgles_types_1.asName)(refTable)} ADD FOREIGN KEY (${(0, prostgles_types_1.asName)(colName)}) REFERENCES ${(0, prostgles_types_1.asName)(tableName)} (id);`;
|
|
99
|
+
console.log(`Referenced file column ${refTable} (${colName}) exists but is not referencing file table. Trying to add REFERENCE constraing...\n${query}`);
|
|
100
|
+
await runQuery(query);
|
|
101
|
+
console.log("SUCCESS: " + query);
|
|
102
|
+
}
|
|
103
|
+
catch (e) {
|
|
104
|
+
console.error(`Could not add constraing. Err: ${e instanceof Error ? e.message : JSON.stringify(e)}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
else {
|
|
108
|
+
console.error(`Referenced file column ${refTable} (${colName}) exists but is not of required type (UUID). Choose a different column name or ALTER the existing column to match the type and the data found in file table ${tableName}(id)`);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
else {
|
|
113
|
+
try {
|
|
114
|
+
const query = `ALTER TABLE ${(0, prostgles_types_1.asName)(refTable)} ADD COLUMN ${(0, prostgles_types_1.asName)(colName)} UUID REFERENCES ${(0, prostgles_types_1.asName)(tableName)} (id);`;
|
|
115
|
+
console.log(`Creating referenced file column ${refTable} (${colName})...\n${query}`);
|
|
116
|
+
await runQuery(query);
|
|
117
|
+
console.log("SUCCESS: " + query);
|
|
118
|
+
}
|
|
119
|
+
catch (e) {
|
|
120
|
+
console.error(`FAILED. Err: ${e instanceof Error ? e.message : JSON.stringify(e)}`);
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
else {
|
|
126
|
+
const lookupTableName = await this.parseSQLIdentifier(`prostgles_lookup_${tableName}_${refTable}`);
|
|
127
|
+
const pKeyFields = cols.filter(f => f.is_pkey);
|
|
128
|
+
if (pKeyFields.length !== 1) {
|
|
129
|
+
console.error(`Could not make link table for ${refTable}. ${pKeyFields} must have exactly one primary key column. Current pkeys: ${pKeyFields.map(f => f.name)}`);
|
|
130
|
+
}
|
|
131
|
+
const pkField = pKeyFields[0];
|
|
132
|
+
const refType = referencedTables[refTable];
|
|
133
|
+
if (!this.dbo[lookupTableName]) {
|
|
134
|
+
// if(!(await dbo[lookupTableName].count())) await db.any(`DROP TABLE IF EXISTS ${lookupTableName};`);
|
|
135
|
+
const action = ` (${tableName} <-> ${refTable}) join table ${lookupTableName}`; // PRIMARY KEY
|
|
136
|
+
const query = `
|
|
137
|
+
CREATE TABLE ${lookupTableName} (
|
|
138
|
+
foreign_id ${pkField.udt_name} ${refType === "one" ? " PRIMARY KEY " : ""} REFERENCES ${(0, prostgles_types_1.asName)(refTable)}(${(0, prostgles_types_1.asName)(pkField.name)}),
|
|
139
|
+
media_id UUID NOT NULL REFERENCES ${(0, prostgles_types_1.asName)(tableName)}(id)
|
|
140
|
+
)
|
|
141
|
+
`;
|
|
142
|
+
console.log(`Creating ${action} ...`, lookupTableName);
|
|
143
|
+
await runQuery(query);
|
|
144
|
+
console.log(`Created ${action}`);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
const cols = await this.dbo[lookupTableName].getColumns();
|
|
148
|
+
const badCols = cols.filter(c => !c.references);
|
|
149
|
+
await Promise.all(badCols.map(async (badCol) => {
|
|
150
|
+
console.error(`Prostgles: media ${lookupTableName} joining table has lost a reference constraint for column ${badCol.name}.` +
|
|
151
|
+
` This may have been caused by a DROP TABLE ... CASCADE.`);
|
|
152
|
+
let q = ` ALTER TABLE ${(0, prostgles_types_1.asName)(lookupTableName)} ADD FOREIGN KEY (${badCol.name}) `;
|
|
153
|
+
console.log("Trying to add the missing constraint back");
|
|
154
|
+
if (badCol.name === "foreign_id") {
|
|
155
|
+
q += `REFERENCES ${(0, prostgles_types_1.asName)(refTable)}(${(0, prostgles_types_1.asName)(pkField.name)}) `;
|
|
156
|
+
}
|
|
157
|
+
else if (badCol.name === "media_id") {
|
|
158
|
+
q += `REFERENCES ${(0, prostgles_types_1.asName)(tableName)}(id) `;
|
|
159
|
+
}
|
|
160
|
+
if (q) {
|
|
161
|
+
try {
|
|
162
|
+
await runQuery(q);
|
|
163
|
+
console.log("Added missing constraint back");
|
|
164
|
+
}
|
|
165
|
+
catch (e) {
|
|
166
|
+
console.error("Failed to add missing constraint", e);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}));
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
await prg.refreshDBO();
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* 4. Serve media through express
|
|
176
|
+
*/
|
|
177
|
+
const { fileServeRoute = `/${tableName}`, expressApp: app } = fileTable;
|
|
178
|
+
if (fileServeRoute.endsWith("/")) {
|
|
179
|
+
throw `fileServeRoute must not end with a '/'`;
|
|
180
|
+
}
|
|
181
|
+
this.fileRoute = fileServeRoute;
|
|
182
|
+
if (app) {
|
|
183
|
+
app.get(this.fileRouteExpress, async (req, res) => {
|
|
184
|
+
if (!this.dbo[tableName]) {
|
|
185
|
+
res.status(500).json({ err: `Internal error: media table (${tableName}) not valid` });
|
|
186
|
+
return false;
|
|
187
|
+
}
|
|
188
|
+
const mediaTable = this.dbo[tableName];
|
|
189
|
+
try {
|
|
190
|
+
const { name } = req.params;
|
|
191
|
+
if (typeof name !== "string" || !name)
|
|
192
|
+
throw "Invalid media name";
|
|
193
|
+
const media = await mediaTable.findOne({ name }, { select: { id: 1, name: 1, signed_url: 1, signed_url_expires: 1, content_type: 1 } }, { httpReq: req });
|
|
194
|
+
if (!media) {
|
|
195
|
+
/**
|
|
196
|
+
* Redirect to login !??
|
|
197
|
+
*/
|
|
198
|
+
// const mediaExists = await mediaTable.count({ name });
|
|
199
|
+
// if(mediaExists && this.prostgles.authHandler){
|
|
200
|
+
// } else {
|
|
201
|
+
// throw "Invalid media";
|
|
202
|
+
// }
|
|
203
|
+
throw "Invalid media";
|
|
204
|
+
}
|
|
205
|
+
if (this.s3Client) {
|
|
206
|
+
let url = media.signed_url;
|
|
207
|
+
const expires = +(media.signed_url_expires || 0);
|
|
208
|
+
const EXPIRES = Date.now() + FileManager_1.HOUR;
|
|
209
|
+
if (!url || expires < EXPIRES) {
|
|
210
|
+
url = await this.getFileS3URL(media.name, 60 * 60);
|
|
211
|
+
await mediaTable.update({ name }, { signed_url: url, signed_url_expires: EXPIRES });
|
|
212
|
+
}
|
|
213
|
+
res.redirect(url);
|
|
214
|
+
}
|
|
215
|
+
else {
|
|
216
|
+
const pth = `${this.config.localFolderPath}/${media.name}`;
|
|
217
|
+
if (!fs.existsSync(pth)) {
|
|
218
|
+
throw new Error("File not found");
|
|
219
|
+
}
|
|
220
|
+
res.contentType(media.content_type);
|
|
221
|
+
res.sendFile(pth);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
catch (e) {
|
|
225
|
+
console.log(e);
|
|
226
|
+
res.status(404).json({ err: "Invalid/missing media" });
|
|
227
|
+
}
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
exports.initFileManager = initFileManager;
|