cognite-neat 0.92.2__py3-none-any.whl → 0.93.0__py3-none-any.whl

cognite/neat/_app/ui/neat-app/src/types/WorkflowTypes.ts

@@ -0,0 +1,388 @@
+export enum WorkflowState {
+    CREATED = "CREATED",
+    RUNNING = "RUNNING",
+    COMPLETED = "COMPLETED",
+    FAILED = "FAILED"
+}
+
+export enum StepExecutionStatus {
+    SUCCESS = "COMPLETED",
+    FAILED = "FAILED",
+    SKIPPED = "SKIPPED",
+    STARTED = "STARTED",
+    UNKNOWN = "UNKNOWN"
+}
+
+export class UIConfig {
+    pos_x: number = 0;
+    pos_y: number = 0;
+}
+
+export class WorkflowConfigItem {
+    name: string;
+    value: string;
+    label?: string;
+    type?: string;
+    required: boolean = false;
+    options?: string[];
+    group?: string;
+}
+
+class StepConfigurable {
+    name: string;
+    value?: string | null;
+    label?: string | null;
+    type?: string | null;  // string , secret , number , boolean , json
+    required: boolean = false;
+    options?: string[] | null;
+
+    constructor(
+        name: string,
+        value: string | null = null,
+        label: string | null = null,
+        type: string | null = null,
+        required: boolean = false,
+        options: string[] | null = null
+    ) {
+        this.name = name;
+        this.value = value;
+        this.label = label;
+        this.type = type;
+        this.required = required;
+        this.options = options;
+    }
+}
+
+
+export class StepMetadata {
+    name: string;
+    category: string = "";
+    description: string = "";
+    input: string[];
+    output: string[];
+    configurables: StepConfigurable[] = [];
+    type: string = "";
+    version: string = "";
+    docs_url: string = "";
+
+    constructor(name: string, input: string[], output: string[]) {
+        this.name = name;
+        this.input = input;
+        this.output = output;
+    }
+}
+
+export class StepRegistry {
+    steps: StepMetadata[] = [];
+    static fromJSON(json: any): StepRegistry {
+        let stepRegistry = new StepRegistry();
+        Object.assign(stepRegistry.steps, json);
+        return stepRegistry;
+    }
+    getStepByName(name: string): StepMetadata {
+        let step = this.steps.find(step => step.name == name);
+        return step;
+    }
+}
+
+
+export class WorkflowStepDefinition {
+    id: string;
+    label?: string;
+    stype?: string;
+    description?: string;
+    method?: string;
+    enabled?: boolean = true;
+    system_component_id?: string;
+    trigger?: boolean = false;
+    max_retries?: number = 0;
+    retry_delay?: number = 3;
+    transition_to?: string[];
+    params?:any = {}
+    ui_config?: UIConfig = new UIConfig();
+    configs?: Map<string, any> = new Map<string, any>();
+    complex_configs?: Map<string, any> = new Map<string, any>();
+}
+
+
+export class WorkflowSystemComponent {
+    id: string;
+    label: string;
+    transition_to?: string[];
+    description?: string;
+    ui_config?: UIConfig = new UIConfig();
+}
+
+export class WorkflowDefinition {
+    name: string;
+    description?: string;
+    steps: WorkflowStepDefinition[];
+    system_components: WorkflowSystemComponent[];
+    configs?: WorkflowConfigItem[];
+    static fromJSON(json: any): WorkflowDefinition {
+        let workflow = new WorkflowDefinition();
+        Object.assign(workflow, json);
+        return workflow;
+    }
+    serializeToJson(): any {
+        let json = JSON.stringify(this);
+        return json;
+    }
+
+    toJSON() {
+        return {
+            name: this.name,
+            description: this.description,
+            configs: this.configs,
+            steps: this.steps,
+            system_components: this.system_components
+        };
+    }
+
+    isNewIdUnique(id: string): boolean {
+        let step = this.steps.find(step => step.id == id);
+        if (step == null)
+            return true;
+        return false;
+    }
+
+    isStepInputConfigured(stepId: string,inputParamName: string ,stepRegistry: StepRegistry): boolean {
+        let step = this.steps.find(step => step.id == stepId);
+        if (step == null)
+            return false;
+        if (step.stype != "stdstep")
+            return false;
+        if (!stepRegistry)
+            return false;
+        let listOfAllOutputs = ["WorkflowConfigs","CdfStore","CogniteClient"];
+        this.steps?.forEach(step => {
+                let outputs = stepRegistry.getStepByName(step.method)?.output;
+                listOfAllOutputs = listOfAllOutputs.concat(outputs);
+        });
+
+        return listOfAllOutputs.includes(inputParamName)
+    }
+
+
+    upsertConfigItem(config: WorkflowConfigItem) {
+        let index = this.configs.findIndex(c => c.name == config.name);
+        if (index >= 0) {
+            this.configs[index] = config;
+        }else {
+            this.configs.push(config);
+        }
+    }
+
+    // insertConfigItemFromTemplate(stepName : string,stepRegistry: StepRegistry) {
+    //     let step_template = stepRegistry.getStepByName(stepName);
+
+
+    //     console.dir(step_template)
+    //     console.dir(this.configs)
+
+
+    //     for (let config of step_template.configurables) {
+    //         let index = this.configs.findIndex(c => c.name == config.name);
+    //         if (index >= 0) {
+    //             continue;
+    //         }else {
+    //             config.group = step_template.name;
+    //             this.configs.push(config);
+    //         }
+    //     }
+    // }
+
+    addConfigItem(config: WorkflowConfigItem) {
+        this.configs.push(config);
+    }
+
+    deleteConfigItem(name: string) {
+        let index = this.configs.findIndex(c => c.name == name);
+        if (index >= 0) {
+            this.configs.splice(index, 1);
+
+        }
+    }
+
+    getStepById(id: string): WorkflowStepDefinition {
+        let step = this.steps.find(step => step.id == id);
+        return step;
+    }
+
+    getSystemComponentById(id: string): WorkflowSystemComponent {
+        let systemComponent = this.system_components.find(systemComponent => systemComponent.id == id);
+        return systemComponent;
+    }
+
+    updateStep(id:string,step: WorkflowStepDefinition) {
+        let index = this.steps.findIndex(s => s.id == id);
+        if (index >= 0) {
+            this.steps[index] = step;
+        }
+    }
+
+    updateSystemComponent(id:string,systemComponent: WorkflowSystemComponent) {
+        let index = this.system_components.findIndex(g => g.id == id);
+        if (index >= 0) {
+            this.system_components[index] = systemComponent;
+        }
+    }
+
+    convertStepsToNodes():any {
+        let nodes = [];
+        this.steps?.forEach(step => {
+            let style = {};
+            if (step.enabled == false) {
+                style = { borderColor: 'red'};
+            }
+            let node = {
+                id: step.id,
+                position: {
+                    x: step.ui_config.pos_x,
+                    y: step.ui_config.pos_y
+                },
+                data: {
+                    label: step.label
+                },
+                width: 150,
+                height: 40,
+                style : style,
+                selected: false,
+                positionAbsolute: {
+                    x: step.ui_config.pos_x,
+                    y: step.ui_config.pos_y
+                },
+                dragging: false
+            }
+            nodes.push(node);
+        });
+        return nodes;
+    }
+
+
+    convertSystemComponentsToNodes():any {
+        let nodes = [];
+        this.system_components?.forEach(step => {
+            let style = {};
+            let node = {
+                id: step.id,
+                position: {
+                    x: step.ui_config.pos_x,
+                    y: step.ui_config.pos_y
+                },
+                data: {
+                    label: step.label
+                },
+                width: 150,
+                height: 40,
+                style : style,
+                selected: false,
+                positionAbsolute: {
+                    x: step.ui_config.pos_x,
+                    y: step.ui_config.pos_y
+                },
+                dragging: false
+            }
+            nodes.push(node);
+        });
+        return nodes;
+    }
+
+    convertStepsToEdges():any {
+        let edges = [];
+        this.steps?.forEach(step => {
+            if (step.transition_to) {
+                step.transition_to.forEach(transition => {
+                    let edge = {
+                        id: step.id + "_" + transition,
+                        source: step.id,
+                        target: transition,
+                        animated: true,
+                        label: "",
+                        arrowHeadType: "arrowclosed"
+                    }
+                    edges.push(edge);
+                });
+            }
+        });
+        return edges;
+    }
+
+    convertSystemComponentsToEdges():any {
+        let edges = [];
+        this.system_components?.forEach(step => {
+            if (step.transition_to) {
+                step.transition_to.forEach(transition => {
+                    let edge = {
+                        id: step.id + "_" + transition,
+                        source: step.id,
+                        target: transition,
+                        animated: true,
+                        label: "",
+                        arrowHeadType: "arrowclosed"
+                    }
+                    edges.push(edge);
+                });
+            }
+        });
+
+        console.dir(edges)
+        return edges;
+    }
+
+    updatePositions(nodes:any) {
+        nodes.forEach(node => {
+            let step = this.steps.find(step => step.id == node.id);
+            if (step) {
+                step.ui_config.pos_x = Math.round(node.position.x);
+                step.ui_config.pos_y = Math.round(node.position.y);
+            }else {
+                let systemComponent = this.system_components.find(systemComponent => systemComponent.id == node.id);
+                if (systemComponent) {
+                    systemComponent.ui_config.pos_x = Math.round(node.position.x);
+                    systemComponent.ui_config.pos_y = Math.round(node.position.y);
+                }
+            }
+        });
+    }
+    // Update step transitions from edges
+    updateStepTransitions(edges:any) {
+        this.steps.forEach(step => {
+            step.transition_to = [];
+        });
+        edges.forEach(edge => {
+            let source = this.steps.find(step => step.id == edge.source);
+            if (source) {
+                source.transition_to.push(edge.target);
+            }
+        });
+    }
+    // Update systemComponent transitions from edges
+    updateSystemComponentTransitions(edges:any) {
+        this.system_components.forEach(systemComponent => {
+            systemComponent.transition_to = [];
+        });
+        edges.forEach(edge => {
+            let source = this.system_components.find(systemComponent => systemComponent.id == edge.source);
+            if (source) {
+                source.transition_to.push(edge.target);
+            }
+        });
+    }
+
+    deleteStep(id:string) {
+        let index = this.steps.findIndex(s => s.id == id);
+        if (index >= 0) {
+            this.steps.splice(index,1);
+        }
+    }
+
+    deleteSystemComponent(id:string) {
+        let index = this.system_components.findIndex(g => g.id == id);
+        if (index >= 0) {
+            this.system_components.splice(index,1);
+        }
+    }
+
+
+}

cognite/neat/_app/ui/neat-app/src/views/AboutView.tsx

@@ -0,0 +1,61 @@
+import React, { useState, useEffect } from 'react';
+import { Typography, List, ListItem, ListItemText } from '@mui/material';
+import { getNeatApiRootUrl } from 'components/Utils';
+
+
+const AboutView = () => {
+  const [structure, setStructure] = useState(null);
+  const [neatApiRootUrl, setNeatApiRootUrl] = useState(getNeatApiRootUrl());
+  useEffect(() => {
+    const fetchData = async () => {
+      try {
+        const response = await fetch(neatApiRootUrl+'/api/about');
+        const data = await response.json();
+        setStructure(data);
+      } catch (error) {
+        console.error('Error fetching data:', error);
+      }
+    };
+
+    fetchData();
+  }, []);
+
+  if (!structure) {
+    return <div>Loading...</div>;
+  }
+
+  return (
+    <div >
+      <Typography variant="h5" >
+        NEAT Version: {structure.version}
+      </Typography>
+      <Typography variant="h6" >
+        Docs URL: <a href='https://thisisneat.io'>https://thisisneat.io</a>
+      </Typography>
+      <Typography variant="h6" >
+        Docs URL: <a href='https://cognite-neat.readthedocs-hosted.com/en/latest/'>https://cognite-neat.readthedocs-hosted.com/en/latest/</a>
+      </Typography>
+      <Typography variant="h6" >
+        Github: <a href='https://github.com/cognitedata/neat'>https://github.com/cognitedata/neat</a>
+      </Typography>
+      <Typography variant="h6" >
+        Issues tracker (Github): <a href='https://github.com/cognitedata/neat/issues'> https://github.com/cognitedata/neat/issues </a>
+      </Typography>
+      <Typography variant="h6" >
+        License (Apache 2.0): <a href='https://github.com/cognitedata/neat/blob/main/LICENSE'> https://github.com/cognitedata/neat/blob/main/LICENSE </a>
+      </Typography>
+      <Typography variant="h5">
+        3rd party packages :
+      </Typography>
+      <List dense={true}>
+        {structure.packages.map((spackage, index) => (
+          <ListItem key={index}>
+            <ListItemText primary={spackage} />
+          </ListItem>
+        ))}
+      </List>
+    </div>
+  );
+};
+
+export default AboutView;

cognite/neat/_app/ui/neat-app/src/views/ConfigView.tsx

@@ -0,0 +1,184 @@
+import * as React from 'react';
+import Box from '@mui/material/Box';
+import Paper from '@mui/material/Paper';
+import Stack from '@mui/material/Stack';
+import DeleteIcon from '@mui/icons-material/Delete';
+import EditIcon from '@mui/icons-material/Edit';
+import { styled } from '@mui/material/styles';
+import { SelectChangeEvent } from '@mui/material/Select';
+import Button from '@mui/material/Button';
+import TextField from '@mui/material/TextField';
+import LinearProgress from '@mui/material/LinearProgress';
+import { useState, useEffect } from 'react';
+import { getNeatApiRootUrl, getSelectedWorkflowName } from 'components/Utils';
+import { WorkflowConfigItem, WorkflowDefinition } from 'types/WorkflowTypes';
+import { Dialog, DialogActions, DialogContent, DialogTitle, FormControl, IconButton, List, ListItem, ListItemText, Table, TableBody, TableCell, TableContainer, TableHead, TableRow } from '@mui/material';
+
+const Item = styled(Paper)(({ theme }) => ({
+  backgroundColor: theme.palette.mode === 'dark' ? '#1A2027' : '#fff',
+  ...theme.typography.body2,
+  padding: theme.spacing(1),
+  textAlign: 'left',
+  color: theme.palette.text.secondary,
+}));
+
+export default function ConfigView() {
+  const [graphSourceName, setGraphSourceName] = React.useState('');
+  const [workflowName, setWorkflowName] = React.useState(getSelectedWorkflowName());
+  const [loading, setLoading] = React.useState(false);
+  const [dialogOpen, setDialogOpen ] = useState(false);
+  const [workflowDefinition, setWorkflowDefinition] = useState<WorkflowDefinition>(new WorkflowDefinition());
+  const [workflowConfigItems, setWorkflowConfigItems] = useState<WorkflowConfigItem[]>([]);
+  const [selectedConfigItem, setSelectedConfigItem] = useState<WorkflowConfigItem>(new WorkflowConfigItem());
+
+  const handleGraphSourceNameChange = (event: SelectChangeEvent) => {
+    setGraphSourceName(event.target.value as string);
+  };
+  const [neatApiRootUrl, setNeatApiRootUrl] = useState(getNeatApiRootUrl());
+
+  useEffect(() => {
+    loadWorkflowConfigs();
+  }, []);
+
+
+  const loadWorkflowConfigs = () => {
+    const workflowName = getSelectedWorkflowName();
+    const url = neatApiRootUrl+"/api/workflow/workflow-definition/" + workflowName;
+    fetch(url).then((response) => response.json()).then((data) => {
+      const workflow = WorkflowDefinition.fromJSON(data.definition);
+      setWorkflowDefinition(workflow);
+      setWorkflowConfigItems(workflow.configs);
+
+    }).catch((error) => {
+      console.error('Error:', error);
+    }).finally(() => { });
+  }
+
+  const saveWorkflow = () => {
+    let wdef = workflowDefinition;
+    wdef.configs = workflowConfigItems;
+    const url = neatApiRootUrl+"/api/workflow/workflow-definition/"+workflowName;
+    fetch(url,{ method:"post",body:wdef.serializeToJson(),headers: {
+        'Content-Type': 'application/json;charset=utf-8'
+    }}).then((response) => response.json()).then((data) => {
+        window.location.reload();
+    }
+    ).catch((error) => {
+        console.error('Error:', error);
+    })
+};
+
+const configEditorDialogHandler = (configItem: WorkflowConfigItem,action: string) => {
+  console.dir(configItem)
+  switch (action) {
+    case "save":
+      workflowDefinition.upsertConfigItem(configItem);
+      break;
+    case "delete":
+      workflowDefinition.deleteConfigItem(configItem.name)
+      break;
+  }
+  setDialogOpen(false);
+}
+
+const openItemEditorDialog = (selectedConfigItem: WorkflowConfigItem) => {
+    if (selectedConfigItem) {
+      setSelectedConfigItem(selectedConfigItem);
+    }else{
+      setSelectedConfigItem(new WorkflowConfigItem());
+    }
+    setDialogOpen(true);
+}
+
+  return (
+    <Box sx={{ width: 1200 }}>
+      <TableContainer component={Paper}>
+      <Table sx={{ minWidth: 650 }}  size="small" aria-label="simple table">
+        <TableHead>
+          <TableRow>
+            <TableCell>Name</TableCell>
+            <TableCell align="right">Config group</TableCell>
+            <TableCell align="right">Value</TableCell>
+            <TableCell align="right">Label</TableCell>
+            <TableCell align="right">Action</TableCell>
+
+          </TableRow>
+        </TableHead>
+        <TableBody>
+          {workflowConfigItems?.map((row) => (
+            <TableRow
+              key={row.name}
+              sx={{ '&:last-child td, &:last-child th': { border: 0 } }}
+            >
+              <TableCell component="th" scope="row">
+               <b> {row.name} </b>
+              </TableCell>
+              <TableCell align="right">{row.group}</TableCell>
+              <TableCell align="right">{row.value}</TableCell>
+              <TableCell align="right">{row.label}</TableCell>
+              <TableCell align="right">
+              <IconButton aria-label="edit" size="small" onClick={ () => openItemEditorDialog(row) } > <EditIcon fontSize="small" /> </IconButton>
+              <IconButton aria-label="delete" size="small" onClick={ ()=> configEditorDialogHandler(row,"delete")}> <DeleteIcon fontSize="small" /> </IconButton>
+              </TableCell>
+            </TableRow>
+          ))}
+        </TableBody>
+      </Table>
+
+    </TableContainer>
+      <Button  variant="outlined" size="small" sx={{marginRight:2,marginTop:2}} onClick={saveWorkflow} >Save</Button>
+      <Button variant="outlined" size="small" sx={{marginRight:2,marginTop:2}} onClick={()=>openItemEditorDialog(null)} >Add</Button>
+      <ConfigEditorDialog open={dialogOpen} onClose={configEditorDialogHandler} configItem={selectedConfigItem} />
+      {loading && (<LinearProgress />)}
+    </Box>
+  );
+}
+
+
+export function ConfigEditorDialog(props: any)
+{
+    const [dialogOpen, setDialogOpen] = useState(false);
+    const [configItem, setConfigItem] = useState<WorkflowConfigItem>(null);
+    const handleDialogSave = () => {
+        setDialogOpen(false);
+        props.onClose(configItem,"save");
+    };
+    const handleDialogCancel = () => {
+        setDialogOpen(false);
+        props.onClose(configItem,"cancel");
+    };
+    const handleConfigItemChange = (name: string, value: any) => {
+
+        console.dir(configItem);
+        let updConfigItem = Object.assign({},configItem);
+        updConfigItem[name] = value;
+
+        console.dir(updConfigItem);
+        setConfigItem(updConfigItem);
+    }
+    useEffect(() => {
+        if (props.open){
+            setDialogOpen(true);
+            setConfigItem(props.configItem);
+            console.dir(props.configItem);
+        }
+      }, [props.open]);
+
+return (
+<Dialog open={dialogOpen} onClose={handleDialogCancel}>
+<DialogTitle>Configuration editor</DialogTitle>
+<DialogContent >
+  <FormControl sx={{ width: 500 }} >
+    <TextField sx={{ marginTop: 1 }} id="step-config-id" fullWidth label="Name" size='small' variant="outlined" value={configItem?.name} onChange={(event) => { handleConfigItemChange("name", event.target.value) }} />
+    <TextField sx={{ marginTop: 1 }} id="step-config-descr" fullWidth label="Value" size='small' variant="outlined" value={configItem?.value} onChange={(event) => { handleConfigItemChange("value", event.target.value) }} />
+    <TextField sx={{ marginTop: 1 }} id="step-config-label" fullWidth label="Label" size='small' variant="outlined" value={configItem?.label} onChange={(event) => { handleConfigItemChange("label", event.target.value) }} />
+    <TextField sx={{ marginTop: 1 }} id="step-config-label" fullWidth label="Config group" size='small' variant="outlined" value={configItem?.group} onChange={(event) => { handleConfigItemChange("group", event.target.value) }} />
+  </FormControl>
+</DialogContent>
+<DialogActions>
+  <Button variant="outlined" size="small" onClick={handleDialogSave}>Save</Button>
+  <Button variant="outlined" size="small" onClick={handleDialogCancel}>Cancel</Button>
+</DialogActions>
+</Dialog>
+)
+}