verteilen-core 1.0.0

package/src/script/execute/base.ts

@@ -0,0 +1,309 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { v6 as uuid6 } from 'uuid';
+import { CronJobState, DataType, ExecuteProxy, ExecuteState, Header, Job, JobCategory, JobType, JobType2, Libraries, Messager, Parameter, Project, Record, Task, WebsocketPack, WorkState } from "../../interface";
+import { WebsocketManager } from "../socket_manager";
+import { Util_Parser } from './util_parser';
+
+/**
+ * The base class of task scheduler, contain some basic funcationality
+ */
+export class ExecuteManager_Base {
+    /**
+     * The task scheduler UUID
+     */
+    uuid: string
+    name: string
+    /**
+     * Register record\
+     * This record holds the project data you want to process
+     */
+    record: Record
+    /**
+     * Current select task\
+     * If it's undefined, it means:
+     * * It's finish the current task
+     * * It has not start processing yet
+     */
+    current_t:Task | undefined = undefined
+    /**
+     * Current select project\
+     * If it's undefined, it means:
+     * * It's finish the current project
+     * * It has not start processing yet
+     */
+    current_p:Project | undefined = undefined
+    /**
+     * The list of projects you want to process\
+     * Each project UUID should be unique by now\
+     * Prevent findIndex error, When there is repeat project source
+     */
+    current_projects:Array<Project> = []
+    /**
+     * The connection nodes list
+     */
+    current_nodes:Array<WebsocketPack> = []
+    /**
+     * Cron job type execute record
+     */
+    current_cron:Array<CronJobState> = []
+    /**
+     * Single job type execute record
+     */
+    current_job:Array<WorkState> = []
+    /**
+     * Current execute task use multithread setting
+     */
+    current_multithread = 1
+    current_task_count = -1
+    /**
+     * * NONE: Not yet start
+     * * RUNNING: In the processing stage
+     * * FINISH: Everything is finish processing
+     */
+    state:ExecuteState = ExecuteState.NONE
+    /**
+     * * NONE: Not yet start
+     * * RUNNING: In the processing stage
+     * * FINISH: Everything is finish processing
+     */
+    t_state:ExecuteState = ExecuteState.NONE 
+    jobstack = 0
+    first = false
+    libs:Libraries | undefined = undefined
+    proxy:ExecuteProxy | undefined = undefined
+    localPara: Parameter | undefined = undefined
+
+    websocket_manager:WebsocketManager
+    messager_log:Messager
+
+    constructor(_name:string, _websocket_manager:WebsocketManager, _messager_log:Messager, _record:Record) {
+        this.name = _name
+        this.uuid = uuid6()
+        this.record = _record
+        this.websocket_manager = _websocket_manager
+        this.messager_log = _messager_log
+    }
+
+    /**
+     * This will let nodes update the parameter and lib
+     * @param target 
+     */
+    protected sync_local_para = (target:Parameter) => {
+        this.current_nodes.forEach(x => this.sync_para(target, x))
+        this.proxy?.updateParameter(target)
+    }
+
+    //#region Helper
+    protected sync_para = (target:Parameter, source:WebsocketPack) => {
+        const h:Header = {
+            name: 'set_parameter',
+            channel: this.uuid,
+            data: target
+        }
+        const h2:Header = {
+            name: 'set_libs',
+            channel: this.uuid,
+            data: this.libs
+        }
+        source.websocket.send(JSON.stringify(h))
+        source.websocket.send(JSON.stringify(h2))
+    }
+    protected release = (source:WebsocketPack) => {
+        const h:Header = {
+            name: 'release',
+            channel: this.uuid,
+            data: 0
+        }
+        source.websocket.send(JSON.stringify(h))
+    }
+    /**
+     * Check all the cronjob is finish or not
+     */
+    protected check_all_cron_end = () => {
+        return this.current_cron.filter(x => !this.check_cron_end(x)).length == 0
+    }
+    /**
+     * Check input cronjob is finish or not
+     * @param cron target cronjob instance
+     */
+    protected check_cron_end = (cron:CronJobState) => {
+        return cron.work.filter(x => x.state == ExecuteState.RUNNING || x.state == ExecuteState.NONE).length == 0
+    }
+    /**
+     * Check current single is finish or not
+     */
+    protected check_single_end = () => {
+        if(this.current_t == undefined) return false
+        return this.current_job.length == this.current_t.jobs.length && 
+            this.current_job.filter(y => y.state == ExecuteState.RUNNING || y.state == ExecuteState.NONE).length == 0
+    }
+    //#endregion
+
+
+    //#region Utility
+    /**
+     * Project format checking
+     * @param projects 
+     * @returns 
+     */
+    protected validation = (projects:Array<Project>):boolean => {
+        if (this.websocket_manager.targets.length == 0) {
+            this.messager_log(`[Execute State] The execute node does not exists`)
+            return false
+        }
+        projects.forEach(x => {
+            x.task.forEach(t => {
+                if(t.cronjob){
+                    const index = x.parameter?.containers.findIndex(x => x.name == t.cronjobKey && x.type == DataType.Number) ?? -1
+                    if(index == -1){
+                        this.messager_log(`[Execute:CronJob] Project ${x.title} (${x.uuid}), Task ${t.title} (${t.uuid}), Has unknoed parameter: \"${t.cronjobKey}\"`)
+                        this.messager_log(`[Execute:CronJob] Cron task registerd key not found`)
+                        return false
+                    }
+                    else if (x.parameter?.containers[index].value == 0){
+                        this.messager_log(`[Execute:CronJob] Project ${x.title} (${x.uuid}), Task ${t.title} (${t.uuid}), Has unknoed parameter: \"${t.cronjobKey}\"`)
+                        this.messager_log(`[Execute:CronJob] Cron task value must bigger than 0`)
+                        return false
+                    }
+                }
+                if(t.cronjob && t.multi){
+                    const index = x.parameter?.containers.findIndex(x => x.name == t.multiKey && x.type == DataType.Number) ?? -1
+                    if(index == -1){
+                        this.messager_log(`[Execute:Multi] Project ${x.title} (${x.uuid}), Task ${t.title} (${t.uuid}), Has unknoed parameter: \"${t.multiKey}\"`)
+                        this.messager_log(`[Execute:Multi] Cron task registerd key not found`)
+                        return false
+                    }
+                    else if (x.parameter?.containers[index].value == 0){
+                        this.messager_log(`[Execute:Multi] Project ${x.title} (${x.uuid}), Task ${t.title} (${t.uuid}), Has unknoed parameter: \"${t.multiKey}\"`)
+                        this.messager_log(`[Execute:Multi] Cron task value must bigger than 0`)
+                        return false
+                    }
+                }
+            })
+        })
+        return true
+    }
+    protected filter_lib = (projects:Array<Project>, lib:Libraries):Libraries => {
+        const r:Libraries = { libs: [] }
+        projects.forEach(x => {
+            x.task.forEach(y => {
+                y.jobs.forEach(z => {
+                    let code = -1
+                    if((z.category == JobCategory.Execution && z.type == JobType.JAVASCRIPT) || (z.category == JobCategory.Condition && z.type == JobType2.JAVASCRIPT)) code = 0
+                    if(code == -1) return
+                    z.string_args.forEach(s1 => {
+                        const target = lib.libs.find(l => l.name == s1)
+                        if(target != undefined) r.libs.push(target)
+                    })
+                })
+            })
+        })
+        return JSON.parse(JSON.stringify(r))
+    }
+    /**
+     * Get the multi-core setting\
+     * Find in the parameter setting
+     * @param key The multi-core-key
+     * @returns 
+     */
+    protected get_task_multi_count = (t:Task):number => {
+        const r = this.get_number(t.multiKey)
+        return r == -1 ? 1 : r
+    }
+    /**
+     * Get the task's cronjob count
+     */
+    public get_task_state_count(t:Task){
+        if(t.setupjob) return this.current_nodes.length
+        if (t.cronjob) return this.get_number(t.cronjobKey)
+        else return 1
+    }
+
+    /**
+     * Find the number in the parameter, this include the expression phrasing
+     * @param key The name key
+     * @param p Project instance
+     * @returns The value, if key cannot be found, it will return -1
+     */
+    protected get_number(key:string){
+        return ExecuteManager_Base.get_number_global(key, this.localPara)
+    }
+
+    static get_number_global(key:string, localPara:Parameter | undefined){
+        const e = ExecuteManager_Base.parameter_update(localPara!)
+        const a = e.replacePara(`%{${key}}%`)
+        return Number(a)
+    }
+
+    /**
+     * Remove dups item in the list
+     * @param arr 
+     * @returns 
+     */
+    protected removeDups = (arr: any[]): any[] => {
+        return [...new Set(arr)];
+    }
+
+    /**
+     * Filter out the idle and connection open nodes
+     * @returns All idle and open connection nodes
+     */
+    protected get_idle = ():Array<WebsocketPack> => {
+        return this.current_nodes.filter(x => this.check_socket_state(x) != ExecuteState.RUNNING && x.websocket.readyState == 1)
+    }
+    /**
+     * Filter out the connection open nodes
+     * @returns All open connection nodes
+     */
+    protected get_idle_open = ():Array<WebsocketPack> => {
+        return this.current_nodes.filter(x => x.websocket.readyState == 1)
+    }
+
+    protected check_socket_state = (target:WebsocketPack) => {
+        return target.current_job.length == 0 ? ExecuteState.NONE : ExecuteState.RUNNING
+    }
+
+    static string_args_transform = (task:Task, job:Job, messager_log:Messager, localPara:Parameter, n:number) => {
+        let e = ExecuteManager_Base.parameter_update(localPara, n)
+        e = ExecuteManager_Base.property_update(task, e)
+
+        for(let i = 0; i < job.string_args.length; i++){
+            const b = job.string_args[i]
+            if(b == null || b == undefined || b.length == 0) continue
+            if(job.category == JobCategory.Execution && job.type == JobType.CREATE_FILE && i == 1) continue
+            job.string_args[i] = e.replacePara(job.string_args[i])
+            //messager_log(`String replace: "${b}" -> "${job.string_args[i]}"`)
+        }
+    }
+
+    static property_update = (task:Task, e:Util_Parser) => {
+        for(let j = 0; j < task.properties.length; j++){
+            const target = task.properties[j];
+            const times = target.deep ? target.deep : 1
+            let act:any = target.expression
+            for(let k = 0; k < times; k++){
+                act = e.replacePara(`%{${act}}%`)
+            }
+            e.paras.push({ key: task.properties[j].name, value: act})
+        }
+        return e
+    }
+
+    static parameter_update = (localPara:Parameter, n?:number) => {
+        const e = new Util_Parser([...Util_Parser.to_keyvalue(localPara)])
+        if(n != undefined){
+            e.paras.push({ key: 'ck', value: n.toString() })
+        }
+        localPara.containers.forEach((c, index) => {
+            if(c.type != DataType.Expression) return
+            c.value = e.replacePara(`%{${c.meta}}%`)
+            e.paras.find(p => p.key == c.name)!.value = c.value
+        })
+        return e
+    }
+    //#endregion
+}

package/src/script/execute/feedback.ts

@@ -0,0 +1,212 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { BusAnalysis, CronJobState, DataType, ExecuteState, FeedBack, Header, Setter, Single, WebsocketPack, WorkState } from "../../interface"
+import { ExecuteManager_Base } from "./base"
+
+/**
+ * Recevied the information from the nodes\
+ * This include job feedback and error feedback and pong and other stuff
+ */
+export class ExecuteManager_Feedback extends ExecuteManager_Base{
+    /**
+     * The analysis method for decoding the information where the nodes is sending
+     * @param d Package info
+     */
+    Analysis = (d:BusAnalysis) => {
+        const targetn = this.current_nodes.find(x => x.uuid == d.c?.uuid)
+        if(targetn == undefined) {
+            this.messager_log("Not inside")
+            return
+        }
+        const typeMap:{ [key:string]:Function } = {
+            'feedback_message': this.feedback_message,
+            'feedback_job': this.feedback_job,
+            'feedback_string': this.feedback_string,
+            'feedback_boolean': this.feedback_boolean,
+            'feedback_number': this.feedback_number,
+            'feedback_object': this.feedback_object,
+        }
+        if(typeMap.hasOwnProperty(d.name)){
+            const castingFunc = typeMap[d.h.name]
+            castingFunc(d.h.data, targetn, d.h.meta)
+        }else{
+            this.messager_log(`[Source Data Analysis] Decode failed, Unknowed header, name: ${d.name}, meta: ${d.h.meta}`)
+        }
+    }
+
+    //#region Feedback
+    /**
+     * Print information, sended by the node worker
+     * @param data feedback data, any type
+     * @param source The node target
+     */
+    private feedback_message = (data:Single, source:WebsocketPack | undefined, meta:string | undefined) => {
+        if(source == undefined) {
+            this.messager_log("[Server Feedback Warn] source is none")
+            return
+        }
+        if(this.state == ExecuteState.NONE) {
+            this.messager_log("[Server Feedback Warn] state is none, should not received feedback")
+            return
+        }
+        this.messager_log(`[Execute] Single Received data: ${data.data}, cron length: ${this.current_cron.length}`)
+        let index = 0
+        if(this.current_cron.length > 0 && meta != undefined){
+            const r = this.GetCronAndWork(meta, source)
+            const cron:CronJobState | undefined = r[0]
+            const work:WorkState | undefined = r[1]
+            if(cron != undefined && work != undefined){
+                index = cron.id
+            }
+        }
+        const d:FeedBack = { 
+            node_uuid: source.uuid,
+            index: index,
+            job_uuid: '',
+            runtime_uuid: '',
+            meta: 0,
+            message: data.data
+        }
+        this.proxy?.feedbackMessage(d)
+    }
+    /**
+     * The job has been finish executing, sended by the node worker
+     * @param data feedback data
+     * @param source The node target
+     */
+    private feedback_job = (data:FeedBack, source:WebsocketPack | undefined) => {
+        if(source == undefined) return
+        if(this.state == ExecuteState.NONE) return
+        this.jobstack = Math.max(this.jobstack - 1, 0)
+        if(this.current_t == undefined) {
+            console.error("Cannot feedback when task is null")
+            return
+        }
+        this.messager_log(`[Execute] Job Feedback: ${data.job_uuid} ${data.runtime_uuid} ${data.message} ${data.meta}`)
+        // If it's a single type work
+        
+        if(this.current_job.length > 0){
+            const work = this.current_job.find(x => x.uuid == source.uuid && x.state == ExecuteState.RUNNING)
+            if(work == undefined) {
+                console.error("Cannot find the feedback container, work", work)
+                return
+            }
+            data.index = 0
+            this.proxy?.executeJobFinish([work.job, 0, source.uuid, data.meta])
+            work.state = data.meta == 0 ? ExecuteState.FINISH : ExecuteState.ERROR
+            if(this.check_single_end()){
+                this.proxy?.executeSubtaskFinish([this.current_t!, 0, source.uuid])
+                this.messager_log(`[Execute] Subtask finish: ${this.current_t!.uuid}`)
+            }
+        }
+        // If it's a cronjob type work
+        else if(this.current_cron.length > 0){
+            const r = this.GetCronAndWork(data.runtime_uuid, source)
+            const cron:CronJobState | undefined = r[0]
+            const work:WorkState | undefined = r[1]
+            if(cron == undefined || work == undefined) {
+                console.error("Cannot find the feedback container, cron or work", data.runtime_uuid, cron, work)
+                console.error("Full current cron instance", this.current_cron)
+                return
+            }
+
+            this.proxy?.executeJobFinish([work.job, cron.id, source.uuid, data.meta])
+            data.index = cron.id
+            work.state = data.meta == 0 ? ExecuteState.FINISH : ExecuteState.ERROR
+            if(this.check_cron_end(cron)){
+                this.proxy?.executeSubtaskFinish([this.current_t, cron.id, cron.uuid ])
+                this.messager_log(`[Execute] Subtask finish: ${this.current_t!.uuid}`)
+                cron.uuid = ''
+            }
+        }
+        // Reset the state of the node
+        const index = source.current_job.findIndex(x => x == data.runtime_uuid)
+        if(index == -1){
+            this.messager_log(`[Execute] Cannot find runtime uuid: ${data.runtime_uuid} in websocket pack source: ${source.uuid}`)
+        }else{
+            source.current_job.splice(index, 1)
+        }
+        data.node_uuid = source.uuid
+        this.proxy?.feedbackMessage(data)
+    }
+    /**
+     * When one of the node decide to change the parameter of string value
+     * @param data The assigner
+     */
+    private feedback_string = (data:Setter) => {
+        if(this.current_p == undefined) return
+        const index = this.localPara!.containers.findIndex(x => x.name == data.key && x.type == DataType.String)
+        if(index != -1) this.localPara!.containers[index].value = data.value
+        else this.localPara!.containers.push({ name: data.key, value: data.value, type: DataType.String, hidden: true, runtimeOnly: true })
+        this.messager_log(`[String Feedback] ${data.key} = ${data.value}`)
+        // Sync to other
+        const d:Header = { name: 'set_parameter', data: this.localPara!}
+        this.current_nodes.forEach(x => x.websocket.send(JSON.stringify(d)))
+        this.proxy?.updateParameter(this.localPara!)
+    }
+    /**
+     * When one of the node decide to change the parameter of number value
+     * @param data The assigner
+     */
+    private feedback_number = (data:Setter) => {
+        if(this.current_p == undefined) return
+        const index = this.localPara!.containers.findIndex(x => x.name == data.key && x.type == DataType.Number)
+        if(index != -1) this.localPara!.containers[index].value = data.value
+        else this.localPara!.containers.push({ name: data.key, value: data.value, type: DataType.Number, hidden: true, runtimeOnly: true })
+        this.messager_log(`[Number Feedback] ${data.key} = ${data.value}`)
+        // Sync to other
+        const d:Header = { name: 'set_parameter', data: this.localPara!}
+        this.current_nodes.forEach(x => x.websocket.send(JSON.stringify(d)))
+        this.proxy?.updateParameter(this.localPara!)
+    }
+    /**
+     * When one of the node decide to change the parameter of object value
+     * @param data The assigner
+     */
+    private feedback_object = (data:Setter) => {
+        if(this.current_p == undefined) return
+        const index = this.localPara!.containers.findIndex(x => x.name == data.key && x.type == DataType.Object)
+        if(index != -1) this.localPara!.containers[index].value = data.value
+        else this.localPara!.containers.push({ name: data.key, value: data.value, type: DataType.Object, hidden: true, runtimeOnly: true })
+        this.messager_log(`[Object Feedback] ${data.key}`)
+        // Sync to other
+        const d:Header = { name: 'set_parameter', data: this.localPara!}
+        this.current_nodes.forEach(x => x.websocket.send(JSON.stringify(d)))
+        this.proxy?.updateParameter(this.localPara!)
+    }
+    /**
+     * When one of the node decide to change the parameter of boolean value
+     * @param data The assigner
+     */
+    private feedback_boolean = (data:Setter) => {
+        if(this.current_p == undefined) return
+        const index = this.localPara!.containers.findIndex(x => x.name == data.key && x.type == DataType.Boolean)
+        if(index != -1) this.localPara!.containers[index].value = data.value
+        else this.localPara!.containers.push({ name: data.key, value: data.value, type: DataType.Boolean, hidden: true, runtimeOnly: true })
+        this.messager_log(`[Boolean Feedback] ${data.key} = ${data.value}`)
+        // Sync to other
+        const d:Header = { name: 'set_parameter', data: this.localPara!}
+        this.current_nodes.forEach(x => x.websocket.send(JSON.stringify(d)))
+        this.proxy?.updateParameter(this.localPara!)
+    }
+
+    private GetCronAndWork = (runtime:string, source:WebsocketPack):[CronJobState | undefined, WorkState | undefined] => {
+        let cron:CronJobState | undefined = undefined
+        let work:WorkState | undefined = undefined
+        const crons = this.current_cron.filter(x => x.uuid == source.uuid)
+        for(let i = 0; i < crons.length; i++){
+            const c = crons[i]
+            const a = c.work.find(x => x.runtime == runtime)
+            if(a != undefined) {
+                cron = c
+                work = a
+                break
+            }
+        }
+        return [cron, work]
+    }
+    //#endregion
+}

package/src/script/execute/region_job.ts

@@ -0,0 +1,14 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { ExecuteManager } from "../execute_manager"
+
+export class Region_Job {
+    target:ExecuteManager
+
+    constructor(_target:ExecuteManager){
+        this.target = _target
+    }
+}

package/src/script/execute/region_project.ts

@@ -0,0 +1,23 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { ExecuteState, Project } from "../../interface"
+import { ExecuteManager } from "../execute_manager"
+import { Region_Task } from "./region_task"
+
+export class Region_Project {
+    target:ExecuteManager
+    state: ExecuteState
+    project: Project
+    task: Region_Task | undefined
+    constructor(_target:ExecuteManager, _project:Project){
+        this.project = _project
+        this.target = _target
+        this.state = _project.task.length > 0 ? ExecuteState.NONE : ExecuteState.SKIP
+        if(_project.task.length > 0){
+            this.task = new Region_Task(_target, _project.task[0])
+        }
+    }
+}

package/src/script/execute/region_subtask.ts

@@ -0,0 +1,14 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { ExecuteManager } from "../execute_manager"
+
+export class Region_Subtask {
+    target:ExecuteManager
+
+    constructor(_target:ExecuteManager){
+        this.target = _target
+    }
+}

package/src/script/execute/region_task.ts

@@ -0,0 +1,23 @@
+// ========================
+//                           
+//      Share Codebase     
+//                           
+// ========================
+import { ExecuteState, Task } from "../../interface";
+import { ExecuteManager } from "../execute_manager";
+import { Region_Subtask } from "./region_subtask";
+
+export class Region_Task {
+    target:ExecuteManager
+    state: ExecuteState
+    task: Task
+    subtask: Region_Subtask | undefined
+    constructor(_target:ExecuteManager, _task:Task){
+        this.task = _task
+        this.target = _target
+        this.state = _task.jobs.length > 0 ? ExecuteState.NONE : ExecuteState.SKIP
+        if(_task.jobs.length > 0){
+            this.subtask = new Region_Subtask(_target)
+        }
+    }
+}