verteilen-core 1.0.0

package/src/client/job_execute.ts

@@ -0,0 +1,216 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import WebSocket from "ws";
+import { Job, JobCategory, JobType, JobType2, JobType2Text, JobTypeText, Libraries, Messager, Messager_log, OnePath, Parameter, PluginList, TwoPath } from "../interface";
+import { i18n } from "../plugins/i18n";
+import { ClientJavascript } from "./javascript";
+import { ClientJobParameter } from "./job_parameter";
+import { ClientOS } from "./os";
+
+/**
+ * The job execute worker\
+ * This class should spawn by the cluster thread to prevent heavy calculation on the main thread
+ */
+export class ClientJobExecute {
+    /**
+     * Project parameters for references
+     */
+    parameter:Parameter | undefined
+    /**
+     * User library for scripts
+     */
+    libraries:Libraries | undefined
+    /**
+     * The job uuid\
+     * This will put in the prefix of message
+     */
+    tag: string
+    runtime:string
+
+    private messager:Messager
+    private messager_log:Messager_log
+    private javascript:ClientJavascript
+    private os:ClientOS
+    private para:ClientJobParameter
+    private job:Job
+    private plugin:PluginList
+
+    constructor(_messager:Messager, _messager_log:Messager_log, _job:Job, _source:WebSocket | undefined, _plugin:PluginList){
+        this.messager = _messager
+        this.messager_log = _messager_log
+        this.tag = _job.uuid
+        this.runtime = _job.runtime_uuid || ''
+        this.job = _job
+        this.plugin = _plugin
+        this.para = new ClientJobParameter()
+        this.os = new ClientOS(() => this.tag, () => this.job.runtime_uuid || '', _messager, _messager_log)
+        this.javascript = new ClientJavascript(_messager, _messager_log, () => this.job)
+        this.parameter = process.env.parameter != undefined ? JSON.parse(process.env.parameter) : undefined
+        this.libraries = process.env.libraries != undefined ? JSON.parse(process.env.libraries) : undefined
+
+        ClientJavascript.Init(_messager, _messager_log, this.os, this.para, 
+            () => this.libraries,
+            () => this.parameter,
+            () => this.job
+        )
+    }
+
+    /**
+     * The entry function to execute the job container
+     * @param job Target job
+     */
+    execute = () => {
+        // Output the job type message to let user know what is going on
+        this.messager_log(`[Execute] ${this.job.uuid}  ${this.job.category == JobCategory.Execution ? i18n.global.t(JobTypeText[this.job.type]) : i18n.global.t(JobType2Text[this.job.type])}`, this.tag, this.runtime)
+        const child = this.job.category == JobCategory.Execution ? this.execute_job_exe() : this.execute_job_con()
+        return child
+    }
+
+    stop_all = () => {
+        this.os.stopall()
+    }
+    
+    /**
+     * Execute the job that classify as run 
+     * @param job Target job
+     * @returns Promise instance
+     */
+    private execute_job_exe = () => {
+        return new Promise<string>(async (resolve, reject) => {
+            switch(this.job.type as JobType){
+                case JobType.COPY_FILE:
+                    {
+                        const data:TwoPath = { from: this.job.string_args[0], to: this.job.string_args[1] }
+                        this.os.file_copy(data)
+                        resolve(`Copy file successfully, ${data.from}, ${data.to}`)
+                        break
+                    }
+                case JobType.COPY_DIR:
+                    {
+                        const data:TwoPath = { from: this.job.string_args[0], to: this.job.string_args[1] }
+                        this.os.dir_copy(data)
+                        resolve(`Copy dir successfully, ${data.from}, ${data.to}`)
+                        break
+                    }
+                case JobType.DELETE_FILE:
+                    {
+                        const data:OnePath = { path: this.job.string_args[0] }
+                        this.os.file_delete(data)
+                        resolve(`Delete file successfully, ${data.path}`)
+                        break
+                    }
+                case JobType.DELETE_DIR:
+                    {
+                        const data:OnePath = { path: this.job.string_args[0] }
+                        this.os.dir_delete(data)
+                        resolve(`Delete folder successfully, ${data.path}`)
+                        break
+                    }
+                case JobType.CREATE_DIR:
+                    {
+                        const data:OnePath = { path: this.job.string_args[0] }
+                        this.os.dir_create(data)
+                        resolve(`Create dir successfully, ${data.path}`)
+                        break
+                    }
+                case JobType.CREATE_FILE:
+                    {
+                        const data:TwoPath = { from: this.job.string_args[0], to: this.job.string_args[1] }
+                        this.os.file_write(data)
+                        resolve(`Create file successfully, ${data.from} ${data.to}`)
+                        break
+                    }
+                case JobType.RENAME:
+                    {
+                        const data:TwoPath = { from: this.job.string_args[0], to: this.job.string_args[1] }
+                        this.os.rename(data)
+                        resolve(`Rename successfully, ${data.from} ${data.to}`)
+                        break
+                    }
+                case JobType.JAVASCRIPT:
+                    {
+                        await this.javascript.JavascriptExecuteWithLib(this.job.script, this.job.string_args).then(() => {
+                            resolve(`Execute Javascript successfully`)
+                        }).catch(k => {
+                            reject(k)
+                        })
+                        break
+                    }
+                case JobType.COMMAND:
+                    {
+                        this.os.command(this.job.string_args[1], this.job.string_args[2], this.job.string_args[0]).then(m => {
+                            resolve(m)
+                        }).catch(err => {
+                            reject(err)
+                        })
+                        break
+                    }
+                case JobType.LIB_COMMAND:
+                    {
+                        const target = this.plugin.plugins.find(x => x.name == this.job.string_args[0])
+                        if(target == undefined){
+                            reject("Cannot find plugin " + this.job.string_args[0])
+                            return
+                        }
+
+                        const archTarget = target.contents.find(x => x.arch == process.arch && x.platform == process.platform)
+                        if(archTarget == undefined){
+                            reject({
+                                code: 1,
+                                message: "Cannot find plugin match arch " + this.job.string_args[0] + "  " + process.arch
+                            })
+                            return
+                        }
+
+                        this.os.lib_command(archTarget.filename, this.job.string_args[1]).then(m => {
+                            resolve(m)
+                        }).catch(err => {
+                            reject(err)
+                        })
+                        break
+                    }
+            }
+        })
+    }
+    
+    /**
+     * Execute the job that classify as condition
+     * @param job Target job
+     * @returns Promise instance
+     */
+    private execute_job_con = () => {
+        return new Promise<string>(async (resolve, reject) => {
+            switch(this.job.type as JobType2){
+                case JobType2.CHECK_PATH:
+                    {
+                        const data:OnePath = { path: this.job.string_args[0] }
+                        if(this.os.fs_exist(data)){
+                            resolve(`Path exist ${data.path}`)
+                        }else{
+                            reject({
+                                code: 2,
+                                message: `Path not exist ${data.path}`
+                            })
+                        }
+                        break
+                    }
+                case JobType2.JAVASCRIPT:
+                    {
+                        const r = await this.javascript.JavascriptExecuteWithLib(this.job.script, this.job.string_args)
+                        if(r != undefined && r == 0){
+                            resolve(`Execute Javascript successfully`)
+                        }else{
+                            reject({
+                                code: 3,
+                                message: `Execute Javascript failed`
+                            })
+                        }
+                        break
+                    }
+            }
+        })
+    }
+}

package/src/client/job_parameter.ts

@@ -0,0 +1,41 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { Header, Setter } from "../interface"
+
+export class ClientJobParameter {
+    /**
+     * Update parameter number on the cluster server
+     * @param data Target KeyValue
+     */
+    feedbacknumber = (data:Setter) => {
+        this.feedback("feedbacknumber", data)
+    }
+    feedbackboolean = (data:Setter) => {
+        this.feedback("feedbackboolean", data)
+    }
+    feedbackstring = (data:Setter) => {
+        this.feedback("feedbackstring", data)
+    }
+    feedbackobject = (data:Setter) => {
+        this.feedback("feedbackobject", data)
+    }
+    feedbacklist = (data:Setter) => {
+        this.feedback("feedbacklist", data)
+    }
+    feedbackselect = (data:Setter) => {
+        this.feedback("feedbackselect", data)
+    }
+    private feedback = (title:string, data:Setter) => {
+        const p:Header = {
+            name: title,
+            data: {
+                key: data.key,
+                value: data.value
+            }
+        }
+        console.log(JSON.stringify(p))
+    }
+}

package/src/client/os.ts

@@ -0,0 +1,210 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { ChildProcess, exec, spawn } from 'child_process';
+import tkill from 'tree-kill'
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import { DATA_FOLDER, Messager, Messager_log, OnePath, TwoPath } from "../interface";
+
+type getstring = ()=>string
+
+/**
+ * The operation system related actions utility\
+ * If you want to do something related to things below
+ * * File operation 
+ * * Folder checker 
+ * * Writing a file
+ * * Call a exe file
+ * 
+ * Please get a instance of this, and call the methods instead using fs youself
+ */
+export class ClientOS {
+    private messager:Messager
+    private messager_log:Messager_log
+    private tag:getstring
+    private runtime:getstring
+    private children:Array<ChildProcess> = []
+
+    /**
+     * 
+     * @param _tag The tag getter that put in the prefix of the message
+     * @param _messager Message method
+     * @param _messager_log Message method with output on the screen feature
+     */
+    constructor(_tag:getstring, _runtime:getstring, _messager:Messager, _messager_log:Messager_log){
+        this.tag = _tag
+        this.runtime = _runtime
+        this.messager = _messager
+        this.messager_log = _messager_log
+    }
+    
+    file_copy = (data:TwoPath) => {
+        this.messager(`[OS Action] File copy, ${data.from} => ${data.to}`, this.tag())
+        fs.copyFileSync(data.from, data.to)
+    }
+    
+    dir_copy = (data:TwoPath) => {
+        this.messager(`[OS Action] Folder copy, ${data.from} => ${data.to}`, this.tag())
+        fs.cpSync(data.from, data.to, { recursive: true, force: true })
+    }
+    
+    file_delete = (data:OnePath) => {
+        this.messager(`[OS Action] File delete, ${data.path}`, this.tag())
+        fs.rmSync(data.path);
+    }
+    
+    dir_delete = (data:OnePath) => {
+        this.messager(`[OS Action] Folder delete, ${data.path}`, this.tag())
+        fs.rmSync(data.path, { recursive: true, force: true })
+    }
+    
+    rename = (data:TwoPath) => {
+        this.messager(`[OS Action] File or dir rename, ${data.from} => ${data.to}`, this.tag())
+        fs.renameSync(data.from, data.to)
+    }
+    
+    fs_exist = (data:OnePath):boolean => {
+        const v = fs.existsSync(data.path)
+        this.messager(`[OS Action] Check path exists, ${data.path}`, this.tag())
+        return v
+    }
+
+    fs_dir_exist = (data:OnePath):boolean => {
+        const p = this.fs_exist(data)
+        if(!p) return false
+        const stat = fs.statSync(data.path)
+        return stat.isDirectory()
+    }
+
+    fs_file_exist = (data:OnePath):boolean => {
+        const p = this.fs_exist(data)
+        if(!p) return false
+        const stat = fs.statSync(data.path)
+        return stat.isFile()
+    }
+    
+    dir_files = (data:OnePath):Array<string> => {
+        const r = fs.readdirSync(data.path, { withFileTypes: true }).filter(x => x.isFile()).map(x => x.name)
+        return r as string[]
+    }
+    
+    dir_dirs = (data:OnePath):Array<string> => {
+        const r = fs.readdirSync(data.path, { withFileTypes: true }).filter(x => x.isDirectory()).map(x => x.name)
+        return r as string[]
+    }
+    
+    dir_create = (data:OnePath) => {
+        this.messager(`[OS Action] Create folder, ${data.path}`, this.tag())
+        fs.mkdirSync(data.path, {recursive: true})
+    }
+    
+    file_write = (data:TwoPath) => {
+        this.messager(`[OS Action] Create file, ${data.from}`, this.tag())
+        fs.writeFileSync(data.from, data.to)
+    }
+    
+    file_read = (data:OnePath) => {
+        return fs.readFileSync(data.path).toString()
+    }
+    
+    /**
+     * Kill all current running processes
+     */
+    stopall = () => {
+        this.children.forEach(x => {
+            x.stdin!.write('q')
+            x.stdin!.end()
+            tkill(x.pid!, 'SIGKILL')
+        })
+        this.children = []
+    }
+    
+    lib_command = async (command:string, args:string):Promise<string> => {
+        const cc = process.platform == "win32" ? command : "./" + command
+        return this.command(cc, args, path.join(os.homedir(), DATA_FOLDER, "exe"))
+    }
+
+    /**
+     * Call command on terminal
+     * @param cwd The system location
+     * @param command Command name, Or you can put filename here
+     * @param args Arguments, It will split by space afterward
+     * @returns 
+     */
+    command = async (command:string, args:string, cwd?:string):Promise<string> => {
+        this.messager_log(`[OS Action] Command cwd: ${cwd}`, this.tag())
+        this.messager_log(`[OS Action] Command command: ${command}`, this.tag())
+        this.messager_log(`[OS Action] Command args: ${args}`, this.tag())
+        return new Promise<string>((resolve, reject) => {
+            const child = spawn(command,  args.split(' '), 
+            { 
+                cwd: cwd, 
+                shell: true, 
+                stdio: ['pipe', 'pipe', 'pipe']
+            })
+            child.stdin.setDefaultEncoding('utf8')
+            // The kill process detecter
+            child.on('spawn', () => {
+                this.children.push(child)
+                this.messager_log(`[Command] Spawn process`, this.tag())
+            })
+            child.on('error', (err) => {
+                this.messager_log(`[Command] Error: ${err}`, this.tag())
+                reject(`Error ${err}`)
+            })
+            child.on('exit', (code, signal) => {
+                this.messager_log(`[Command] Process Exit: ${code}`, this.tag())
+            })
+            child.on('message', (message, sendHandle) => {
+                this.messager_log(`[Command] : ${message.toString()}`, this.tag())
+            })
+            child.on('close', (code, signal) => {
+                this.messager_log(`[Command] Process Close: ${code}`, this.tag())
+                const index = this.children.findIndex(x => x.pid == child.pid)
+                if(index != -1) this.children.splice(index, 1)
+                resolve(`Successfully ${code}`)
+            })
+            child.stdout.setEncoding('utf8');
+            child.stdout.on('data', (chunk) => {
+                this.messager_log(`[Command Info] : ${chunk.toString()}`, this.tag())
+            })
+            child.stderr.setEncoding('utf8');
+            child.stderr.on('data', (chunk) => {
+                this.messager_log(`[Command Error] : ${chunk.toString()}`, this.tag())
+            })
+        })
+    }
+
+    command_sync = async (command:string, args:string, cwd?:string):Promise<string> => {
+        return this.command(command, args, cwd)
+    }
+
+    command_exec = (command:string, args:string, cwd?:string) => {
+        this.messager_log(`[OS Action] Command cwd: ${cwd}`, this.tag())
+        this.messager_log(`[OS Action] Command command: ${command}`, this.tag())
+        this.messager_log(`[OS Action] Command args: ${args}`, this.tag())
+        const child = exec(`${command} ${args}`, { 
+                cwd: cwd
+        })
+
+        child.on('spawn', () => {
+            this.messager_log(`[Command] Spawn process`, this.tag())
+        })
+        child.on('error', (err) => {
+            this.messager_log(`[Command] Error: ${err}`, this.tag())
+        })
+        child.on('exit', (code, signal) => {
+            this.messager_log(`[Command] Process Exit: ${code}`, this.tag())
+        })
+        child.on('message', (message, sendHandle) => {
+            this.messager_log(`[Command] : ${message.toString()}`, this.tag())
+        })
+        child.on('close', (code, signal) => {
+            this.messager_log(`[Command] Process Close: ${code}`, this.tag())
+        })
+    }
+}

package/src/client/parameter.ts

@@ -0,0 +1,58 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import WebSocket from "ws";
+import { Header, Setter } from "../interface";
+
+/**
+ * The parameter feedback helper\
+ * Update the main parameter container on the cluster server
+ */
+export class ClientParameter {
+    private source:WebSocket | undefined
+
+    constructor(_source:WebSocket | undefined){
+        this.source = _source
+    }
+
+    /**
+     * Update parameter number on the cluster server
+     * @param data Target KeyValue
+     */
+    feedbacknumber = (data:Setter) => {
+        this.feedback("feedback_number", data)
+    }
+    /**
+     * Update parameter boolean on the cluster server
+     * @param data Target KeyValue
+     */
+    feedbackboolean = (data:Setter) => {
+        this.feedback("feedback_boolean", data)
+    }
+    /**
+     * Update parameter string on the cluster server
+     * @param data Target KeyValue
+     */
+    feedbackstring = (data:Setter) => {
+        this.feedback("feedback_string", data)
+    }
+    feedbackobject = (data:Setter) => {
+        this.feedback("feedback_object", data)
+    }
+    feedbacklist = (data:Setter) => {
+        this.feedback("feedback_list", data)
+    }
+    feedbackselect = (data:Setter) => {
+        this.feedback("feedback_select", data)
+    }
+    private feedback = (title:string, data:Setter) => {
+        if(this.source == undefined) return
+        const p:Header = {
+            name: title,
+            data: data
+        }
+        this.source.send(JSON.stringify(p, null, 2))
+    }
+}

package/src/client/resource.ts

@@ -0,0 +1,121 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import si from "systeminformation"
+import { ResourceType } from "../interface"
+import { SystemLoad } from "../interface/struct"
+
+/**
+ * The resource query helper
+ */
+export class ClientResource {
+    is_query = false
+
+    Query = async (src:SystemLoad | undefined = undefined, type:ResourceType = ResourceType.ALL):Promise<SystemLoad> => {
+        this.is_query = true
+        const result:SystemLoad = src != undefined ? src : this.create_new()
+
+        const _system = (type & ResourceType.SYSTEM ) == ResourceType.SYSTEM ? si.system() : undefined
+        const _cpu = si.cpu()
+        const _ram = si.mem()
+        const _battery = si.battery()
+        const _load = si.currentLoad()
+        const _os = si.osInfo()
+        const _gpu = si.graphics()
+        const _disk = si.fsSize()
+        const _net = si.networkStats()
+        this.is_query = false
+        
+        return await Promise.all([_system, _cpu, _ram, _battery, _load, _os, _gpu, _disk, _net]).then(x => {
+            const system = x[0]
+            const cpu = x[1]
+            const ram = x[2]
+            const battery = x[3]
+            const load = x[4]
+            const os = x[5]
+            const gpu = x[6]
+            const disk = x[7]
+            const net = x[8]
+
+            if(system != undefined){
+                result.system_name = `${system.manufacturer} ${system.model}`
+                result.virtual = system.virtual
+            }
+            if(os != undefined){
+                result.platform = process.platform
+                result.arch = process.arch
+                result.hostname = os.hostname
+            }
+            if(cpu != undefined){
+                result.cpu_name = `${cpu.manufacturer} ${cpu.brand} ${cpu.speed}`
+                result.cpu_core = cpu.cores
+            }
+            if(load != undefined){
+                result.cpu_usage = load.currentLoadGuest + load.currentLoadIrq + load.currentLoadSystem + load.currentLoad + load.currentLoadSteal + load.currentLoadNice
+            }
+            if(ram != undefined){
+                result.ram_usage = ram.used
+                result.ram_free = ram.free
+                result.ram_total = ram.total
+            }
+            if(battery != undefined){
+                result.battery = battery.hasBattery ? battery.percent : 1
+                result.charging = battery.isCharging
+            }
+            if(gpu != undefined){
+                result.gpu = gpu.controllers.map(x => ({
+                    gpu_name: `${x.vendor} ${x.model}`
+                }))
+            }
+            if(disk != undefined){
+                result.disk = disk.map(x => ({
+                    disk_name: x.fs,
+                    disk_type: x.type,
+                    disk_free: x.available,
+                    disk_total: x.size,
+                    disk_usage: x.used,
+                    disk_percentage: x.use,
+                }))
+            }
+            if(net != undefined){
+                result.net = net.map(x => ({
+                    net_name: x.iface,
+                    download: x.rx_sec,
+                    upload: x.tx_sec
+                }))
+            }
+
+            result.pid_usage = process.pid
+            return result
+        })
+    }
+
+    private create_new = ():SystemLoad => {
+        return {
+            system_name: '',
+            virtual: false,
+            platform: '',
+            arch: '',
+            hostname: '',
+        
+            cpu_name: '',
+            cpu_core: 0,
+            cpu_usage: 0,
+        
+            ram_usage: 0,
+            ram_free: 0,
+            ram_total: 0,
+        
+            battery: 0,
+            charging: false,
+        
+            gpu: [],
+            disk: [],
+            net: [],
+        
+            pid_usage: 0
+        }
+    }
+}