prostgles-server 4.0.16 → 4.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/AuthHandler.js +1 -1
- package/dist/AuthHandler.js.map +1 -1
- package/dist/DBSchemaBuilder.js +3 -2
- package/dist/DBSchemaBuilder.js.map +1 -1
- package/dist/DboBuilder/QueryBuilder/QueryBuilder.d.ts +19 -14
- package/dist/DboBuilder/QueryBuilder/QueryBuilder.d.ts.map +1 -1
- package/dist/DboBuilder/TableHandler.d.ts +9 -7
- package/dist/DboBuilder/TableHandler.d.ts.map +1 -1
- package/dist/DboBuilder/TableHandler.js +1 -1
- package/dist/DboBuilder/TableHandler.js.map +1 -1
- package/dist/DboBuilder/ViewHandler.d.ts +3 -3
- package/dist/DboBuilder/ViewHandler.d.ts.map +1 -1
- package/dist/DboBuilder/getSubscribeRelatedTables.d.ts +1 -1
- package/dist/DboBuilder/getSubscribeRelatedTables.d.ts.map +1 -1
- package/dist/DboBuilder/parseUpdateRules.d.ts.map +1 -1
- package/dist/DboBuilder/parseUpdateRules.js.map +1 -1
- package/dist/DboBuilder.js +1 -1
- package/dist/DboBuilder.js.map +1 -1
- package/dist/FileManager/FileManager.d.ts +143 -0
- package/dist/FileManager/FileManager.d.ts.map +1 -0
- package/dist/FileManager/FileManager.js +273 -0
- package/dist/FileManager/FileManager.js.map +1 -0
- package/dist/FileManager/initFileManager.d.ts +4 -0
- package/dist/FileManager/initFileManager.d.ts.map +1 -0
- package/dist/FileManager/initFileManager.js +209 -0
- package/dist/FileManager/initFileManager.js.map +1 -0
- package/dist/FileManager/parseFile.d.ts +15 -0
- package/dist/FileManager/parseFile.d.ts.map +1 -0
- package/dist/FileManager/parseFile.js +59 -0
- package/dist/FileManager/parseFile.js.map +1 -0
- package/dist/FileManager/upload.d.ts +6 -0
- package/dist/FileManager/upload.d.ts.map +1 -0
- package/dist/FileManager/upload.js +76 -0
- package/dist/FileManager/upload.js.map +1 -0
- package/dist/FileManager/uploadStream.d.ts +5 -0
- package/dist/FileManager/uploadStream.d.ts.map +1 -0
- package/dist/FileManager/uploadStream.js +70 -0
- package/dist/FileManager/uploadStream.js.map +1 -0
- package/dist/PostgresNotifListenManager.js +1 -1
- package/dist/PostgresNotifListenManager.js.map +1 -1
- package/dist/Prostgles.d.ts +2 -2
- package/dist/Prostgles.d.ts.map +1 -1
- package/dist/Prostgles.js +8 -5
- package/dist/Prostgles.js.map +1 -1
- package/dist/PubSubManager/PubSubManager.d.ts +3 -3
- package/dist/PubSubManager/PubSubManager.js +1 -1
- package/dist/PubSubManager/PubSubManager.js.map +1 -1
- package/dist/TableConfig/TableConfig.d.ts.map +1 -1
- package/dist/TableConfig/TableConfig.js +12 -7
- package/dist/TableConfig/TableConfig.js.map +1 -1
- package/lib/AuthHandler.js +1 -1
- package/lib/AuthHandler.ts +1 -1
- package/lib/DBSchemaBuilder.js +3 -2
- package/lib/DBSchemaBuilder.ts +3 -2
- package/lib/DboBuilder/TableHandler.d.ts +4 -2
- package/lib/DboBuilder/TableHandler.d.ts.map +1 -1
- package/lib/DboBuilder/TableHandler.js +1 -1
- package/lib/DboBuilder/TableHandler.ts +2 -2
- package/lib/DboBuilder/getSubscribeRelatedTables.d.ts +1 -1
- package/lib/DboBuilder/getSubscribeRelatedTables.d.ts.map +1 -1
- package/lib/DboBuilder/getSubscribeRelatedTables.ts +1 -1
- package/lib/DboBuilder/parseUpdateRules.d.ts.map +1 -1
- package/lib/DboBuilder/parseUpdateRules.ts +2 -2
- package/lib/{FileManager.d.ts → FileManager/FileManager.d.ts} +12 -20
- package/lib/FileManager/FileManager.d.ts.map +1 -0
- package/lib/FileManager/FileManager.js +303 -0
- package/lib/FileManager/FileManager.ts +378 -0
- package/lib/FileManager/initFileManager.d.ts +4 -0
- package/lib/FileManager/initFileManager.d.ts.map +1 -0
- package/lib/FileManager/initFileManager.js +231 -0
- package/lib/FileManager/initFileManager.ts +236 -0
- package/lib/FileManager/parseFile.d.ts +15 -0
- package/lib/FileManager/parseFile.d.ts.map +1 -0
- package/lib/FileManager/parseFile.js +58 -0
- package/lib/FileManager/parseFile.ts +69 -0
- package/lib/FileManager/upload.d.ts +6 -0
- package/lib/FileManager/upload.d.ts.map +1 -0
- package/lib/FileManager/upload.js +98 -0
- package/lib/FileManager/upload.ts +83 -0
- package/lib/FileManager/uploadStream.d.ts +5 -0
- package/lib/FileManager/uploadStream.d.ts.map +1 -0
- package/lib/FileManager/uploadStream.js +92 -0
- package/lib/FileManager/uploadStream.ts +80 -0
- package/lib/Prostgles.d.ts +2 -2
- package/lib/Prostgles.d.ts.map +1 -1
- package/lib/Prostgles.js +8 -5
- package/lib/Prostgles.ts +10 -6
- package/lib/TableConfig/TableConfig.d.ts.map +1 -1
- package/lib/TableConfig/TableConfig.js +12 -7
- package/lib/TableConfig/TableConfig.ts +13 -8
- package/package.json +3 -3
- package/tests/client/PID.txt +1 -1
- package/tests/client/package-lock.json +31 -4561
- package/tests/client/package.json +2 -2
- package/tests/client/tsconfig.json +1 -0
- package/tests/server/package-lock.json +1151 -13
- package/tests/server/package.json +3 -1
- package/tests/server/tsconfig.json +1 -0
- package/tsconfig.json +1 -0
- package/lib/FileManager.d.ts.map +0 -1
- package/lib/FileManager.js +0 -676
- package/lib/FileManager.ts +0 -811
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
+
};
|
|
28
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
+
exports.bytesToSize = exports.getFileType = exports.getFileTypeFromFilename = exports.removeExpressRoute = exports.FileManager = exports.asSQLIdentifier = exports.HOUR = void 0;
|
|
30
|
+
const aws_sdk_1 = require("aws-sdk");
|
|
31
|
+
// import { PutObjectCommand, S3 } from "@aws-sdk/client-s3";
|
|
32
|
+
const fs = __importStar(require("fs"));
|
|
33
|
+
const sharp = __importStar(require("sharp"));
|
|
34
|
+
const check_disk_space_1 = __importDefault(require("check-disk-space"));
|
|
35
|
+
const prostgles_types_1 = require("prostgles-types");
|
|
36
|
+
const aws_sdk_2 = __importDefault(require("aws-sdk"));
|
|
37
|
+
const path = __importStar(require("path"));
|
|
38
|
+
const initFileManager_1 = require("./initFileManager");
|
|
39
|
+
const parseFile_1 = require("./parseFile");
|
|
40
|
+
const upload_1 = require("./upload");
|
|
41
|
+
const uploadStream_1 = require("./uploadStream");
|
|
42
|
+
exports.HOUR = 3600 * 1000;
|
|
43
|
+
const asSQLIdentifier = async (name, db) => {
|
|
44
|
+
return (await db.one("select format('%I', $1) as name", [name]))?.name;
|
|
45
|
+
};
|
|
46
|
+
exports.asSQLIdentifier = asSQLIdentifier;
|
|
47
|
+
class FileManager {
|
|
48
|
+
static testCredentials = async (accessKeyId, secretAccessKey) => {
|
|
49
|
+
const sts = new aws_sdk_2.default.STS();
|
|
50
|
+
aws_sdk_2.default.config.credentials = {
|
|
51
|
+
accessKeyId,
|
|
52
|
+
secretAccessKey
|
|
53
|
+
};
|
|
54
|
+
const ident = await sts.getCallerIdentity({}).promise();
|
|
55
|
+
return ident;
|
|
56
|
+
};
|
|
57
|
+
s3Client;
|
|
58
|
+
config;
|
|
59
|
+
imageOptions;
|
|
60
|
+
prostgles;
|
|
61
|
+
get dbo() {
|
|
62
|
+
if (!this.prostgles?.dbo) {
|
|
63
|
+
// this.prostgles?.refreshDBO();
|
|
64
|
+
throw "this.prostgles.dbo missing";
|
|
65
|
+
}
|
|
66
|
+
return this.prostgles.dbo;
|
|
67
|
+
}
|
|
68
|
+
get db() {
|
|
69
|
+
if (!this.prostgles?.db)
|
|
70
|
+
throw "this.prostgles.db missing";
|
|
71
|
+
return this.prostgles.db;
|
|
72
|
+
}
|
|
73
|
+
tableName;
|
|
74
|
+
fileRoute;
|
|
75
|
+
get fileRouteExpress() {
|
|
76
|
+
return this.fileRoute + "/:name";
|
|
77
|
+
}
|
|
78
|
+
checkInterval;
|
|
79
|
+
constructor(config, imageOptions) {
|
|
80
|
+
this.config = config;
|
|
81
|
+
this.imageOptions = imageOptions;
|
|
82
|
+
if ("region" in config) {
|
|
83
|
+
const { region, accessKeyId, secretAccessKey } = config;
|
|
84
|
+
this.s3Client = new aws_sdk_1.S3({
|
|
85
|
+
region,
|
|
86
|
+
credentials: { accessKeyId, secretAccessKey },
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
const fullConfig = this.prostgles?.opts.fileTable;
|
|
90
|
+
if (fullConfig?.delayedDelete) {
|
|
91
|
+
this.checkInterval = setInterval(async () => {
|
|
92
|
+
const fileTable = fullConfig.tableName;
|
|
93
|
+
const daysDelay = fullConfig.delayedDelete?.deleteAfterNDays ?? 0;
|
|
94
|
+
if (fileTable && this.dbo[fileTable]?.delete && daysDelay) {
|
|
95
|
+
const filesToDelete = await this.dbo[fileTable]?.find?.({ deleted_from_storage: null, deleted: { ">": Date.now() - (daysDelay * exports.HOUR * 24) } }) ?? [];
|
|
96
|
+
for await (const file of filesToDelete) {
|
|
97
|
+
await this.deleteFile(file.name);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
console.error("FileManager checkInterval delayedDelete FAIL: Could not access file table tableHandler.delete()");
|
|
102
|
+
}
|
|
103
|
+
}, Math.max(10000, (fullConfig.delayedDelete.checkIntervalHours || 0) * exports.HOUR));
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
async getFileStream(name) {
|
|
107
|
+
if ("bucket" in this.config && this.s3Client) {
|
|
108
|
+
return this.s3Client.getObject({ Key: name, Bucket: this.config.bucket }).createReadStream();
|
|
109
|
+
}
|
|
110
|
+
else if ("localFolderPath" in this.config) {
|
|
111
|
+
const filePath = path.resolve(`${this.config.localFolderPath}/${name}`);
|
|
112
|
+
if (!fs.existsSync(filePath)) {
|
|
113
|
+
throw `File ${filePath} could not be found`;
|
|
114
|
+
}
|
|
115
|
+
return fs.createReadStream(filePath, { encoding: undefined });
|
|
116
|
+
}
|
|
117
|
+
else
|
|
118
|
+
throw new Error("Not expected");
|
|
119
|
+
}
|
|
120
|
+
async deleteFile(name) {
|
|
121
|
+
if ("bucket" in this.config && this.s3Client) {
|
|
122
|
+
const res = await this.s3Client?.deleteObject({ Bucket: this.config.bucket, Key: name }).promise();
|
|
123
|
+
return res;
|
|
124
|
+
}
|
|
125
|
+
else if ("localFolderPath" in this.config) {
|
|
126
|
+
const path = `${this.config.localFolderPath}/${name}`;
|
|
127
|
+
if (!fs.existsSync(path)) {
|
|
128
|
+
throw `File ${path} could not be found`;
|
|
129
|
+
}
|
|
130
|
+
fs.unlinkSync(path);
|
|
131
|
+
if (fs.existsSync(path))
|
|
132
|
+
throw new Error("Could not delete file");
|
|
133
|
+
}
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
parseFile = parseFile_1.parseFile.bind(this);
|
|
137
|
+
getLocalFileUrl = (name) => this.fileRoute ? `${this.fileRoute}/${name}` : "";
|
|
138
|
+
checkFreeSpace = async (folderPath, fileSize = 0) => {
|
|
139
|
+
if (!this.s3Client && "localFolderPath" in this.config) {
|
|
140
|
+
const { minFreeBytes = 1.048e6 } = this.config;
|
|
141
|
+
const required = Math.max(fileSize, minFreeBytes);
|
|
142
|
+
if (required) {
|
|
143
|
+
const diskSpace = await (0, check_disk_space_1.default)(folderPath);
|
|
144
|
+
if (diskSpace.free < required) {
|
|
145
|
+
const err = `There is not enough space on the server to save files.\nTotal: ${bytesToSize(diskSpace.size)} \nRemaning: ${bytesToSize(diskSpace.free)} \nRequired: ${bytesToSize(required)}`;
|
|
146
|
+
throw new Error(err);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
};
|
|
151
|
+
uploadStream = uploadStream_1.uploadStream.bind(this);
|
|
152
|
+
upload = upload_1.upload.bind(this);
|
|
153
|
+
uploadAsMedia = async (params) => {
|
|
154
|
+
const { item, imageOptions } = params;
|
|
155
|
+
const { name, data, content_type, extension } = item;
|
|
156
|
+
if (!data)
|
|
157
|
+
throw "No file provided";
|
|
158
|
+
if (!name || typeof name !== "string")
|
|
159
|
+
throw "Expecting a string name";
|
|
160
|
+
// const type = await this.getMIME(data, name, allowedExtensions, dissallowedExtensions);
|
|
161
|
+
let _data = data;
|
|
162
|
+
/** Resize/compress/remove exif from photos */
|
|
163
|
+
if (content_type.startsWith("image") && extension.toLowerCase() !== "gif") {
|
|
164
|
+
const compression = imageOptions?.compression;
|
|
165
|
+
if (compression) {
|
|
166
|
+
console.log("Resizing image");
|
|
167
|
+
let opts;
|
|
168
|
+
if ("contain" in compression) {
|
|
169
|
+
opts = {
|
|
170
|
+
fit: sharp.fit.contain,
|
|
171
|
+
...compression.contain
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
else if ("inside" in compression) {
|
|
175
|
+
opts = {
|
|
176
|
+
fit: sharp.fit.inside,
|
|
177
|
+
...compression.inside
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
_data = await sharp(data)
|
|
181
|
+
.resize(opts)
|
|
182
|
+
.withMetadata(Boolean(imageOptions?.keepMetadata))
|
|
183
|
+
// .jpeg({ quality: 80 })
|
|
184
|
+
.toBuffer();
|
|
185
|
+
}
|
|
186
|
+
else if (!imageOptions?.keepMetadata) {
|
|
187
|
+
/**
|
|
188
|
+
* Remove exif
|
|
189
|
+
*/
|
|
190
|
+
// const metadata = await simg.metadata();
|
|
191
|
+
// const simg = await sharp(data);
|
|
192
|
+
_data = await sharp(data).clone().withMetadata({
|
|
193
|
+
exif: {}
|
|
194
|
+
})
|
|
195
|
+
.toBuffer();
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
const res = await this.upload(_data, name, content_type);
|
|
199
|
+
return res;
|
|
200
|
+
};
|
|
201
|
+
async getFileS3URL(fileName, expiresInSeconds = 30 * 60) {
|
|
202
|
+
const params = {
|
|
203
|
+
Bucket: this.config.bucket,
|
|
204
|
+
Key: fileName,
|
|
205
|
+
Expires: Math.round(expiresInSeconds || 30 * 60) // Error if float
|
|
206
|
+
};
|
|
207
|
+
return await this.s3Client?.getSignedUrlPromise("getObject", params);
|
|
208
|
+
}
|
|
209
|
+
parseSQLIdentifier = async (name) => (0, exports.asSQLIdentifier)(name, this.prostgles.db); // this.prostgles.dbo.sql<"value">("select format('%I', $1)", [name], { returnType: "value" } )
|
|
210
|
+
getColInfo = (args) => {
|
|
211
|
+
const { colName, tableName } = args;
|
|
212
|
+
const tableConfig = this.prostgles?.opts.fileTable?.referencedTables?.[tableName];
|
|
213
|
+
const isReferencingFileTable = this.dbo[tableName]?.columns?.some(c => c.name === colName && c.references && c.references?.some(({ ftable }) => ftable === this.tableName));
|
|
214
|
+
if (isReferencingFileTable) {
|
|
215
|
+
if (tableConfig && typeof tableConfig !== "string") {
|
|
216
|
+
return tableConfig.referenceColumns[colName];
|
|
217
|
+
}
|
|
218
|
+
return { acceptedContent: "*" };
|
|
219
|
+
}
|
|
220
|
+
return undefined;
|
|
221
|
+
};
|
|
222
|
+
init = initFileManager_1.initFileManager.bind(this);
|
|
223
|
+
destroy = () => {
|
|
224
|
+
(0, exports.removeExpressRoute)(this.prostgles?.opts.fileTable?.expressApp, [this.fileRouteExpress]);
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
exports.FileManager = FileManager;
|
|
228
|
+
const removeExpressRoute = (app, routePaths) => {
|
|
229
|
+
const routes = app?._router?.stack;
|
|
230
|
+
if (routes) {
|
|
231
|
+
routes.forEach((route, i) => {
|
|
232
|
+
if (routePaths.filter(prostgles_types_1.isDefined).includes(route.route?.path)) {
|
|
233
|
+
routes.splice(i, 1);
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
};
|
|
238
|
+
exports.removeExpressRoute = removeExpressRoute;
|
|
239
|
+
const getFileTypeFromFilename = (fileName) => {
|
|
240
|
+
const nameParts = fileName.split(".");
|
|
241
|
+
if (nameParts.length < 2)
|
|
242
|
+
return undefined;
|
|
243
|
+
const nameExt = nameParts.at(-1).toLowerCase(), mime = (0, prostgles_types_1.getKeys)(prostgles_types_1.CONTENT_TYPE_TO_EXT).find(k => prostgles_types_1.CONTENT_TYPE_TO_EXT[k].includes(nameExt));
|
|
244
|
+
if (!mime)
|
|
245
|
+
return undefined;
|
|
246
|
+
return {
|
|
247
|
+
mime,
|
|
248
|
+
ext: nameExt,
|
|
249
|
+
};
|
|
250
|
+
};
|
|
251
|
+
exports.getFileTypeFromFilename = getFileTypeFromFilename;
|
|
252
|
+
// const fileType = require("file-type");
|
|
253
|
+
// const res = await fileType.fromBuffer(typeof file === "string"? Buffer.from(file, 'utf8') : file);
|
|
254
|
+
const getFileType = async (file, fileName) => {
|
|
255
|
+
const { fileTypeFromBuffer } = await eval('import("file-type")');
|
|
256
|
+
const fileNameMime = (0, exports.getFileTypeFromFilename)(fileName);
|
|
257
|
+
if (!fileNameMime?.ext)
|
|
258
|
+
throw new Error("File name must contain extenion");
|
|
259
|
+
const res = await fileTypeFromBuffer(typeof file === "string" ? Buffer.from(file, 'utf8') : file);
|
|
260
|
+
if (!res) {
|
|
261
|
+
/* Set correct/missing extension */
|
|
262
|
+
const nameExt = fileNameMime?.ext;
|
|
263
|
+
if (["xml", "txt", "csv", "tsv", "svg", "sql"].includes(nameExt) && fileNameMime.mime) {
|
|
264
|
+
return fileNameMime;
|
|
265
|
+
}
|
|
266
|
+
throw new Error("Could not get the file type from file buffer");
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
if (!res.ext || fileNameMime?.ext.toLowerCase() !== res.ext.toLowerCase()) {
|
|
270
|
+
throw new Error(`There is a mismatch between file name extension and actual buffer extension: ${fileNameMime?.ext} vs ${res.ext}`);
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
return res;
|
|
274
|
+
};
|
|
275
|
+
exports.getFileType = getFileType;
|
|
276
|
+
function bytesToSize(bytes) {
|
|
277
|
+
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
|
278
|
+
if (bytes == 0)
|
|
279
|
+
return '0 Byte';
|
|
280
|
+
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)) + "");
|
|
281
|
+
return (bytes / Math.pow(1024, i)).toFixed(1) + ' ' + sizes[i];
|
|
282
|
+
}
|
|
283
|
+
exports.bytesToSize = bytesToSize;
|
|
284
|
+
/**
|
|
285
|
+
*
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
// if(content_type && typeof content_type !== "string") throw "Invalid content_type provided";
|
|
289
|
+
// if(title && typeof title !== "string") throw "Invalid title provided";
|
|
290
|
+
// let fExt = name.split(".").pop()
|
|
291
|
+
// if(content_type && name.split(".").length > 1 && fExt && fExt.length <= 4){
|
|
292
|
+
// type = {
|
|
293
|
+
// mime: content_type,
|
|
294
|
+
// ext: fExt,
|
|
295
|
+
// fileName: name,
|
|
296
|
+
// }
|
|
297
|
+
// } else {
|
|
298
|
+
// type = await this.getMIME(data, name);//, ["png", "jpg", "ogg", "webm", "pdf", "doc"]);
|
|
299
|
+
// }
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
*/
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
|
|
2
|
+
import { S3, IAM } from 'aws-sdk';
|
|
3
|
+
// import { PutObjectCommand, S3 } from "@aws-sdk/client-s3";
|
|
4
|
+
import * as fs from 'fs';
|
|
5
|
+
import * as stream from 'stream';
|
|
6
|
+
|
|
7
|
+
import * as sharp from "sharp";
|
|
8
|
+
import checkDiskSpace from 'check-disk-space';
|
|
9
|
+
|
|
10
|
+
import { DB, DBHandlerServer, ExpressApp, Prostgles } from '../Prostgles';
|
|
11
|
+
import { ALLOWED_CONTENT_TYPE, ALLOWED_EXTENSION, CONTENT_TYPE_TO_EXT, getKeys, isDefined, ValidatedColumnInfo } from 'prostgles-types';
|
|
12
|
+
|
|
13
|
+
import AWS from 'aws-sdk';
|
|
14
|
+
import * as path from "path";
|
|
15
|
+
import { initFileManager } from "./initFileManager";
|
|
16
|
+
import { parseFile } from "./parseFile";
|
|
17
|
+
import { upload } from "./upload";
|
|
18
|
+
import { uploadStream } from "./uploadStream";
|
|
19
|
+
|
|
20
|
+
export const HOUR = 3600 * 1000;
|
|
21
|
+
|
|
22
|
+
export const asSQLIdentifier = async (name: string, db: DB): Promise<string> => {
|
|
23
|
+
return (await db.one("select format('%I', $1) as name", [name]))?.name
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export type OnProgress = (progress: S3.ManagedUpload.Progress) => void
|
|
27
|
+
|
|
28
|
+
export type ImageOptions = {
|
|
29
|
+
keepMetadata?: boolean;
|
|
30
|
+
compression?:
|
|
31
|
+
/**
|
|
32
|
+
* Will resize image maintaing scale ratio
|
|
33
|
+
*/
|
|
34
|
+
| { inside: { width: number; height: number } }
|
|
35
|
+
| { contain:
|
|
36
|
+
| { width: number }
|
|
37
|
+
| { height: number }
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export type S3Config = {
|
|
42
|
+
region: string;
|
|
43
|
+
bucket: string;
|
|
44
|
+
accessKeyId: string;
|
|
45
|
+
secretAccessKey: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export type LocalConfig = {
|
|
49
|
+
/**
|
|
50
|
+
* example: path.join(__dirname+'/media')
|
|
51
|
+
* note that this location will be relative to the compiled file location
|
|
52
|
+
*/
|
|
53
|
+
localFolderPath: string;
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Minimum amount of free bytes available to allow saving files
|
|
57
|
+
* Defaults to 100MB
|
|
58
|
+
*/
|
|
59
|
+
minFreeBytes?: number;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export type UploadItem = {
|
|
63
|
+
name: string;
|
|
64
|
+
content_type: string;
|
|
65
|
+
data: Buffer;
|
|
66
|
+
extension: string;
|
|
67
|
+
};
|
|
68
|
+
export type UploadedItem = {
|
|
69
|
+
/**
|
|
70
|
+
* Url that is passed to client
|
|
71
|
+
*/
|
|
72
|
+
url: string;
|
|
73
|
+
filePath?: string;
|
|
74
|
+
etag: string;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* S3 url of the resource
|
|
78
|
+
*/
|
|
79
|
+
s3_url?: string;
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Total uploaded file size in bytes
|
|
83
|
+
*/
|
|
84
|
+
content_length: number;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
export class FileManager {
|
|
88
|
+
|
|
89
|
+
static testCredentials = async (accessKeyId: string, secretAccessKey: string) => {
|
|
90
|
+
const sts = new AWS.STS();
|
|
91
|
+
AWS.config.credentials = {
|
|
92
|
+
accessKeyId,
|
|
93
|
+
secretAccessKey
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const ident = await sts.getCallerIdentity({}).promise();
|
|
97
|
+
return ident;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
s3Client?: S3;
|
|
101
|
+
|
|
102
|
+
config: S3Config | LocalConfig;
|
|
103
|
+
imageOptions?: ImageOptions;
|
|
104
|
+
|
|
105
|
+
prostgles?: Prostgles;
|
|
106
|
+
get dbo(): DBHandlerServer {
|
|
107
|
+
if(!this.prostgles?.dbo) {
|
|
108
|
+
// this.prostgles?.refreshDBO();
|
|
109
|
+
throw "this.prostgles.dbo missing"
|
|
110
|
+
}
|
|
111
|
+
return this.prostgles.dbo
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
get db(): DB {
|
|
115
|
+
if(!this.prostgles?.db) throw "this.prostgles.db missing"
|
|
116
|
+
return this.prostgles.db
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
tableName?: string;
|
|
120
|
+
|
|
121
|
+
fileRoute?: string;
|
|
122
|
+
get fileRouteExpress (){
|
|
123
|
+
return this.fileRoute + "/:name";
|
|
124
|
+
}
|
|
125
|
+
private checkInterval?: NodeJS.Timeout;
|
|
126
|
+
|
|
127
|
+
constructor(config: FileManager["config"], imageOptions?: ImageOptions){
|
|
128
|
+
this.config = config;
|
|
129
|
+
this.imageOptions = imageOptions;
|
|
130
|
+
|
|
131
|
+
if("region" in config){
|
|
132
|
+
const { region, accessKeyId, secretAccessKey } = config;
|
|
133
|
+
this.s3Client = new S3({
|
|
134
|
+
region,
|
|
135
|
+
credentials: { accessKeyId, secretAccessKey },
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const fullConfig = this.prostgles?.opts.fileTable;
|
|
140
|
+
if(fullConfig?.delayedDelete){
|
|
141
|
+
this.checkInterval = setInterval(async () => {
|
|
142
|
+
const fileTable = fullConfig.tableName;
|
|
143
|
+
const daysDelay = fullConfig.delayedDelete?.deleteAfterNDays ?? 0;
|
|
144
|
+
if(fileTable && this.dbo[fileTable]?.delete && daysDelay){
|
|
145
|
+
const filesToDelete = await this.dbo[fileTable]?.find?.({ deleted_from_storage: null, deleted: { ">": Date.now() - (daysDelay * HOUR * 24) } }) ?? [];
|
|
146
|
+
for await(const file of filesToDelete){
|
|
147
|
+
await this.deleteFile(file.name)
|
|
148
|
+
}
|
|
149
|
+
} else {
|
|
150
|
+
console.error("FileManager checkInterval delayedDelete FAIL: Could not access file table tableHandler.delete()")
|
|
151
|
+
}
|
|
152
|
+
}, Math.max(10000, (fullConfig.delayedDelete.checkIntervalHours || 0) * HOUR))
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async getFileStream(name: string): Promise<stream.Readable> {
|
|
157
|
+
if("bucket" in this.config && this.s3Client){
|
|
158
|
+
return this.s3Client.getObject({ Key: name, Bucket: this.config.bucket }).createReadStream()
|
|
159
|
+
} else if("localFolderPath" in this.config){
|
|
160
|
+
const filePath = path.resolve(`${this.config.localFolderPath}/${name}`);
|
|
161
|
+
if(!fs.existsSync(filePath)){
|
|
162
|
+
throw `File ${filePath} could not be found`;
|
|
163
|
+
}
|
|
164
|
+
return fs.createReadStream(filePath, { encoding: undefined })
|
|
165
|
+
} else throw new Error("Not expected");
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
async deleteFile(name: string) {
|
|
169
|
+
if("bucket" in this.config && this.s3Client){
|
|
170
|
+
const res = await this.s3Client?.deleteObject({ Bucket: this.config.bucket, Key: name }).promise();
|
|
171
|
+
return res;
|
|
172
|
+
} else if("localFolderPath" in this.config){
|
|
173
|
+
const path = `${this.config.localFolderPath}/${name}`;
|
|
174
|
+
if(!fs.existsSync(path)){
|
|
175
|
+
throw `File ${path} could not be found`;
|
|
176
|
+
}
|
|
177
|
+
fs.unlinkSync(path);
|
|
178
|
+
if(fs.existsSync(path)) throw new Error("Could not delete file");
|
|
179
|
+
}
|
|
180
|
+
return true
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
parseFile = parseFile.bind(this);
|
|
184
|
+
|
|
185
|
+
getLocalFileUrl = (name: string) => this.fileRoute? `${this.fileRoute}/${name}` : "";
|
|
186
|
+
|
|
187
|
+
checkFreeSpace = async (folderPath: string, fileSize = 0) => {
|
|
188
|
+
if(!this.s3Client && "localFolderPath" in this.config) {
|
|
189
|
+
const { minFreeBytes = 1.048e6 } = this.config;
|
|
190
|
+
const required = Math.max(fileSize, minFreeBytes)
|
|
191
|
+
if(required){
|
|
192
|
+
const diskSpace = await checkDiskSpace(folderPath);
|
|
193
|
+
if(diskSpace.free < required){
|
|
194
|
+
const err = `There is not enough space on the server to save files.\nTotal: ${bytesToSize(diskSpace.size)} \nRemaning: ${bytesToSize(diskSpace.free)} \nRequired: ${bytesToSize(required)}`
|
|
195
|
+
throw new Error(err);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
uploadStream = uploadStream.bind(this);
|
|
202
|
+
|
|
203
|
+
upload = upload.bind(this);
|
|
204
|
+
|
|
205
|
+
uploadAsMedia = async (params: {
|
|
206
|
+
item: UploadItem;
|
|
207
|
+
allowedExtensions?: Array<ALLOWED_EXTENSION>;
|
|
208
|
+
dissallowedExtensions?: Array<ALLOWED_EXTENSION>;
|
|
209
|
+
imageOptions?: ImageOptions;
|
|
210
|
+
}): Promise<UploadedItem> => {
|
|
211
|
+
const { item, imageOptions } = params;
|
|
212
|
+
const { name, data, content_type, extension } = item;
|
|
213
|
+
if(!data) throw "No file provided";
|
|
214
|
+
if(!name || typeof name !== "string") throw "Expecting a string name";
|
|
215
|
+
|
|
216
|
+
// const type = await this.getMIME(data, name, allowedExtensions, dissallowedExtensions);
|
|
217
|
+
|
|
218
|
+
let _data = data;
|
|
219
|
+
|
|
220
|
+
/** Resize/compress/remove exif from photos */
|
|
221
|
+
if(content_type.startsWith("image") && extension.toLowerCase() !== "gif"){
|
|
222
|
+
|
|
223
|
+
const compression = imageOptions?.compression
|
|
224
|
+
if(compression){
|
|
225
|
+
console.log("Resizing image")
|
|
226
|
+
let opts;
|
|
227
|
+
if("contain" in compression){
|
|
228
|
+
opts = {
|
|
229
|
+
fit: sharp.fit.contain,
|
|
230
|
+
...compression.contain
|
|
231
|
+
}
|
|
232
|
+
} else if("inside" in compression){
|
|
233
|
+
opts = {
|
|
234
|
+
fit: sharp.fit.inside,
|
|
235
|
+
...compression.inside
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
_data = await sharp(data)
|
|
239
|
+
.resize(opts as any)
|
|
240
|
+
.withMetadata(Boolean(imageOptions?.keepMetadata) as any)
|
|
241
|
+
// .jpeg({ quality: 80 })
|
|
242
|
+
.toBuffer()
|
|
243
|
+
} else if(!imageOptions?.keepMetadata) {
|
|
244
|
+
/**
|
|
245
|
+
* Remove exif
|
|
246
|
+
*/
|
|
247
|
+
// const metadata = await simg.metadata();
|
|
248
|
+
// const simg = await sharp(data);
|
|
249
|
+
|
|
250
|
+
_data = await sharp(data).clone().withMetadata({
|
|
251
|
+
exif: {}
|
|
252
|
+
})
|
|
253
|
+
.toBuffer()
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
const res = await this.upload(_data, name, content_type);
|
|
258
|
+
|
|
259
|
+
return res
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
async getFileS3URL(fileName: string, expiresInSeconds: number = 30 * 60){
|
|
263
|
+
const params = {
|
|
264
|
+
Bucket: (this.config as S3Config).bucket,
|
|
265
|
+
Key: fileName,
|
|
266
|
+
Expires: Math.round(expiresInSeconds || 30 * 60) // Error if float
|
|
267
|
+
};
|
|
268
|
+
return await this.s3Client?.getSignedUrlPromise("getObject", params);
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
parseSQLIdentifier = async (name: string ) => asSQLIdentifier(name, this.prostgles!.db!);// this.prostgles.dbo.sql<"value">("select format('%I', $1)", [name], { returnType: "value" } )
|
|
272
|
+
|
|
273
|
+
getColInfo = (args: { tableName: string; colName: string }): ValidatedColumnInfo["file"] | undefined => {
|
|
274
|
+
const { colName, tableName } = args;
|
|
275
|
+
const tableConfig = this.prostgles?.opts.fileTable?.referencedTables?.[tableName];
|
|
276
|
+
const isReferencingFileTable = this.dbo[tableName]?.columns?.some(c => c.name === colName && c.references && c.references?.some(({ ftable }) => ftable === this.tableName ));
|
|
277
|
+
if(isReferencingFileTable){
|
|
278
|
+
if(tableConfig && typeof tableConfig !== "string"){
|
|
279
|
+
return tableConfig.referenceColumns[colName];
|
|
280
|
+
}
|
|
281
|
+
return { acceptedContent: "*" };
|
|
282
|
+
}
|
|
283
|
+
return undefined;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
init = initFileManager.bind(this);
|
|
287
|
+
|
|
288
|
+
destroy = () => {
|
|
289
|
+
removeExpressRoute(this.prostgles?.opts.fileTable?.expressApp, [this.fileRouteExpress]);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
export const removeExpressRoute = (app: ExpressApp | undefined, routePaths: (string | undefined)[]) => {
|
|
294
|
+
|
|
295
|
+
const routes = app?._router?.stack;
|
|
296
|
+
if(routes){
|
|
297
|
+
routes.forEach((route, i) => {
|
|
298
|
+
if(routePaths.filter(isDefined).includes(route.route?.path as any)){
|
|
299
|
+
routes.splice(i, 1);
|
|
300
|
+
}
|
|
301
|
+
})
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export const getFileTypeFromFilename = (fileName: string): { mime: ALLOWED_CONTENT_TYPE; ext: ALLOWED_EXTENSION | string } | undefined => {
|
|
306
|
+
|
|
307
|
+
const nameParts = fileName.split(".");
|
|
308
|
+
|
|
309
|
+
if(nameParts.length < 2) return undefined;
|
|
310
|
+
|
|
311
|
+
const nameExt = nameParts.at(-1)!.toLowerCase(),
|
|
312
|
+
mime = getKeys(CONTENT_TYPE_TO_EXT).find(k => (CONTENT_TYPE_TO_EXT[k] as readonly string[]).includes(nameExt));
|
|
313
|
+
|
|
314
|
+
if(!mime) return undefined;
|
|
315
|
+
|
|
316
|
+
return {
|
|
317
|
+
mime,
|
|
318
|
+
ext: nameExt as ALLOWED_EXTENSION,
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// const fileType = require("file-type");
|
|
323
|
+
// const res = await fileType.fromBuffer(typeof file === "string"? Buffer.from(file, 'utf8') : file);
|
|
324
|
+
|
|
325
|
+
export const getFileType = async (file: Buffer | string, fileName: string): Promise<{ mime: ALLOWED_CONTENT_TYPE; ext: ALLOWED_EXTENSION }> => {
|
|
326
|
+
|
|
327
|
+
const { fileTypeFromBuffer } = await (eval('import("file-type")') as Promise<typeof import('file-type')>);
|
|
328
|
+
|
|
329
|
+
const fileNameMime = getFileTypeFromFilename(fileName);
|
|
330
|
+
if(!fileNameMime?.ext) throw new Error("File name must contain extenion")
|
|
331
|
+
const res = await fileTypeFromBuffer(typeof file === "string"? Buffer.from(file, 'utf8') : file);
|
|
332
|
+
|
|
333
|
+
if(!res) {
|
|
334
|
+
|
|
335
|
+
/* Set correct/missing extension */
|
|
336
|
+
const nameExt = fileNameMime?.ext;
|
|
337
|
+
if(["xml", "txt", "csv", "tsv", "svg", "sql"].includes(nameExt) && fileNameMime.mime){
|
|
338
|
+
return fileNameMime as any;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
throw new Error("Could not get the file type from file buffer");
|
|
342
|
+
} else {
|
|
343
|
+
|
|
344
|
+
if(!res.ext || fileNameMime?.ext.toLowerCase() !== res.ext.toLowerCase()){
|
|
345
|
+
throw new Error(`There is a mismatch between file name extension and actual buffer extension: ${fileNameMime?.ext } vs ${res.ext}`)
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
return res as any;
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
export function bytesToSize(bytes: number) {
|
|
352
|
+
const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB'];
|
|
353
|
+
if (bytes == 0) return '0 Byte';
|
|
354
|
+
const i = parseInt(Math.floor(Math.log(bytes) / Math.log(1024)) + "");
|
|
355
|
+
return (bytes / Math.pow(1024, i)).toFixed(1) + ' ' + sizes[i];
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
/**
|
|
360
|
+
*
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
// if(content_type && typeof content_type !== "string") throw "Invalid content_type provided";
|
|
364
|
+
// if(title && typeof title !== "string") throw "Invalid title provided";
|
|
365
|
+
// let fExt = name.split(".").pop()
|
|
366
|
+
// if(content_type && name.split(".").length > 1 && fExt && fExt.length <= 4){
|
|
367
|
+
// type = {
|
|
368
|
+
// mime: content_type,
|
|
369
|
+
// ext: fExt,
|
|
370
|
+
// fileName: name,
|
|
371
|
+
// }
|
|
372
|
+
// } else {
|
|
373
|
+
// type = await this.getMIME(data, name);//, ["png", "jpg", "ogg", "webm", "pdf", "doc"]);
|
|
374
|
+
// }
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
*/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"initFileManager.d.ts","sourceRoot":"","sources":["initFileManager.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAqB,MAAM,eAAe,CAAC;AAI/D,wBAAsB,eAAe,CAAC,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,iBAmOtE"}
|