Flowfile 0.2.2__py3-none-any.whl

flowfile_core/schemas/models.py

@@ -0,0 +1,193 @@
+from pydantic import BaseModel
+from typing import List, Any, Callable
+from datetime import datetime
+from dataclasses import dataclass, field
+import os
+import mimetypes
+from copy import deepcopy
+from flowfile_core.configs.settings import FILE_LOCATION
+
+
+class DirItem:
+    name: str
+    full_path: str
+    path: str
+    type: str
+    stats: os.stat = None
+    creation_date: datetime = None
+    access_date: datetime = None
+    modification_date: datetime = None
+    source_path: str = None
+
+    def __init__(self, name: str, path: str,
+                 stats: os.stat = None,
+                 creation_date: datetime = None,
+                 modification_date: datetime = None,
+                 access_date: datetime = None, *args, **kwargs):
+        self.full_path = os.path.relpath(path)
+        if name == FILE_LOCATION:
+            self.name = self.full_path
+            self.source_path = self.full_path
+        else:
+            self.name = name
+            self.source_path = os.sep.join(os.path.split(self.full_path)[:-1])
+        self.path = self.full_path
+        self.stats = os.stat(self.full_path) if stats is None else stats
+        self.creation_date = datetime.fromtimestamp(self.stats.st_ctime) if creation_date is None else creation_date
+        self.modification_date = datetime.fromtimestamp(self.stats.st_mtime) if modification_date is None else modification_date
+        self.access_date = datetime.fromtimestamp(self.stats.st_atime) if access_date is None else access_date
+
+
+@dataclass
+class DirFile(DirItem):
+    ext: str
+    file_size: int
+    size: int
+    mimetype: str = None
+
+    def __init__(self, name: str, path: str):
+        ext = os.path.splitext(name)[-1]
+        if ext == '':
+            ext = 'unk'
+        self.ext = ext
+        self.type = 'file'
+        self.mimetype = mimetypes.guess_type(path)[0]
+        super().__init__(name, path)
+        self.file_size = self.stats.st_size
+        self.size = self.file_size_in_kb
+
+
+    @property
+    def file_size_in_kb(self) -> int:
+        return int(self.file_size/1024)
+
+    def json_repr(self):
+        self_dict = deepcopy(self.__dict__)
+        self_dict.pop('stats')
+        return self_dict
+
+@dataclass
+class DirLocation(DirItem):
+    all_items: List[str] = field(default_factory=list)
+    number_of_items: int = -1
+    files: List[DirFile] = field(default_factory=list)
+    directories: List["DirLocation"] = field(default_factory=list)
+    __size: int = -1
+
+    def __init__(self, name: str, path: str = None):
+        path = '.' + os.path.sep + name if path is None else path
+        self.all_items = os.listdir(path)
+        self.files = []
+        self.directories = []
+        self.type = 'dir'
+        super().__init__(name, path)
+        self.create_structure()
+        self.number_of_items = len(self.all_items)
+
+    def as_dict(self):
+        return {**self.__dict__,**{'size':self.size}}
+
+    def get_dirs(self):
+        return self.directories
+
+    @property
+    def size(self) -> int:
+        if self.__size<0:
+            size = 0
+            for file in self.files:
+                size += file.size
+            for d in self.directories:
+                size += d.size
+            self.__size = size
+        return self.__size
+
+    def create_structure(self):
+        for item_name in self.all_items:
+
+            ref = os.path.join(self.full_path, item_name)
+            if os.path.isdir(ref):
+                self.directories.append(self.__class__(item_name,ref))
+            elif os.path.isfile(ref):
+                self.files.append(DirFile(item_name,ref))
+
+    def json_repr(self):
+        self_dict = deepcopy(self.__dict__)
+        self_dict.pop('stats')
+        files: List[DirFile] = self_dict.pop('files')
+        directories: List[DirLocation] = self_dict.pop('directories')
+        self_dict.pop('all_items')
+        self_dict['files'] = [f.json_repr() for f in files]
+        self_dict['directories'] = [d.json_repr() for d in directories]
+        return self_dict
+
+class Country(BaseModel):
+    id: int
+    name: str
+    abbreviation: str
+
+
+class Address(BaseModel):
+    street: str = None
+    house_number: int = None
+    house_number_addition: str = None
+    city: str = None
+    zipcode: str = None
+    country: Country = None
+
+
+class Team(Address):
+    id: int
+    name: str
+
+
+class User(Address):
+    user_name: str
+    hashed_password: str
+    user_id: int
+    team: Team
+
+
+class ExampleValues(BaseModel):
+    pass
+
+
+class SourceStats(BaseModel):
+    id: int
+    created_by: User
+    created_date: datetime = datetime.now()
+    updated_by: User
+    updated_date: datetime = datetime.now()
+
+
+class Source(SourceStats):
+    name: str
+    team: Team
+
+
+class Table(SourceStats):
+    name: str
+    table_type: str = None
+    table_fields: List['TableField'] = None
+
+
+class TableField(BaseModel):
+    id: int
+    source: Source
+    name: str
+    python_type: type
+
+
+class FieldMapper(SourceStats):
+    mapping_type: str = None  # direct, formula, default value
+    input_fields: List[TableField] = None
+    output_fields: List[TableField]
+    source_table: Table
+    target_table: Table
+    formula: Callable = None
+    default_value: Any = None
+
+
+class TableMapper(SourceStats):
+    source_table: Table
+    target_table: Table
+

flowfile_core/schemas/output_model.py

@@ -0,0 +1,115 @@
+from typing import List, Dict, Optional, Any
+from pydantic import BaseModel, Field
+from datetime import datetime
+import time
+
+
+class NodeResult(BaseModel):
+    node_id: int
+    node_name: str = None
+    start_timestamp: float = Field(default_factory=time.time)
+    end_timestamp: float = 0
+    success: Optional[bool] = None
+    error: str = ''
+    run_time: int = -1
+    is_running: bool = True
+
+
+class RunInformation(BaseModel):
+    flow_id: int
+    start_time: Optional[datetime] = Field(default_factory=datetime.now)
+    end_time: Optional[datetime] = None
+    success: bool
+    nodes_completed: int = 0
+    number_of_nodes: int = 0
+    node_step_result: List[NodeResult]
+
+
+class BaseItem(BaseModel):
+    name: str
+    path: str
+    size: Optional[int] = None
+    creation_date: Optional[datetime] = None
+    access_date: Optional[datetime] = None
+    modification_date: Optional[datetime] = None
+    source_path: Optional[str] = None
+    number_of_items: int = -1
+
+
+class FileColumn(BaseModel):
+    name: str
+    data_type: str
+    is_unique: bool
+    max_value: str
+    min_value: str
+    number_of_empty_values: int
+    number_of_filled_values: int
+    number_of_unique_values: int
+    size: int
+
+
+class TableExample(BaseModel):
+    node_id: int
+    number_of_records: int
+    number_of_columns: int
+    name: str
+    table_schema: List[FileColumn]
+    columns: List[str]
+    data: Optional[List[Dict]] = {}
+
+
+class NodeData(BaseModel):
+    flow_id: int
+    node_id: int
+    flow_type: str
+    left_input: Optional[TableExample] = None
+    right_input: Optional[TableExample] = None
+    main_input: Optional[TableExample] = None
+    main_output: Optional[TableExample] = None
+    left_output: Optional[TableExample] = None
+    right_output: Optional[TableExample] = None
+    has_run: bool = False
+    is_cached: bool = False
+    setting_input: Any = None
+
+
+class OutputFile(BaseItem):
+    ext: Optional[str] = None
+    mimetype: Optional[str] = None
+
+
+class OutputFiles(BaseItem):
+    files: List[OutputFile] = Field(default_factory=list)
+
+
+class OutputTree(OutputFiles):
+    directories: List[OutputFiles] = Field(default_factory=list)
+
+
+class ItemInfo(OutputFile):
+    id: int = -1
+    type: str
+    analysis_file_available: bool = False
+    analysis_file_location: str = None
+    analysis_file_error: str = None
+
+
+class OutputDir(BaseItem):
+    all_items: List[str]
+    items: List[ItemInfo]
+
+
+class ExpressionRef(BaseModel):
+    name: str
+    doc: Optional[str]
+
+
+class ExpressionsOverview(BaseModel):
+    expression_type: str
+    expressions: List[ExpressionRef]
+
+
+class InstantFuncResult(BaseModel):
+    success: Optional[bool] = None
+    result: str
+

flowfile_core/schemas/schemas.py

@@ -0,0 +1,106 @@
+from pydantic import BaseModel, field_validator, ConfigDict
+from typing import List, Dict, Tuple, Iterable, Optional, Any, Literal
+
+
+ExecutionModeLiteral = Literal['Development', 'Performance']
+ExecutionLocationsLiteral = Literal['auto', 'local', 'remote']
+
+
+class FlowSettings(BaseModel):
+    flow_id: int
+    description: Optional[str] = None
+    save_location: Optional[str] = None
+    auto_save: bool = False
+    name: str = ''
+    modified_on: Optional[float] = None
+    path: str
+    execution_mode: ExecutionModeLiteral = 'Performance'
+    show_detailed_progress: bool = True
+    is_running: bool = False
+    is_canceled: bool = False
+    execution_location: ExecutionLocationsLiteral = "auto"
+
+
+class RawLogInput(BaseModel):
+    flowfile_flow_id: int
+    log_message: str
+    log_type: Literal["INFO", "ERROR"]
+    extra: Optional[dict] = None
+
+
+class NodeTemplate(BaseModel):
+    name: str
+    item: str
+    input: int
+    output: int
+    image: str
+    multi: bool = False
+    node_group: str
+    prod_ready: bool = True
+    can_be_start: bool = False
+
+
+class NodeInformation(BaseModel):
+    id: Optional[int] = None
+    type: Optional[str] = None
+    is_setup: Optional[bool] = None
+    description: Optional[str] = ''
+    x_position: Optional[int] = 0
+    y_position: Optional[int] = 0
+    left_input_id: Optional[int] = None
+    right_input_id: Optional[int] = None
+    input_ids: Optional[List[int]] = [-1]
+    outputs: Optional[List[int]] = [-1]
+    setting_input: Optional[Any] = None
+
+    @property
+    def data(self):
+        return self.setting_input
+
+    @property
+    def main_input_ids(self):
+        return self.input_ids
+
+
+class FlowInformation(BaseModel):
+    flow_id: int
+    flow_name: Optional[str] = ''
+    flow_settings: FlowSettings
+    data: Dict[int, NodeInformation] = {}
+    node_starts: List[int]
+    node_connections: List[Tuple[int, int]] = []
+
+    @field_validator('flow_name', mode="before")
+    def ensure_string(cls, v):
+        return str(v) if v is not None else ''
+
+
+class NodeInput(NodeTemplate):
+    id: int
+    pos_x: float
+    pos_y: float
+
+
+class NodeEdge(BaseModel):
+    model_config = ConfigDict(coerce_numbers_to_str=True)
+    id: str
+    source: str
+    target: str
+    targetHandle: str
+    sourceHandle: str
+
+
+class VueFlowInput(BaseModel):
+    node_edges: List[NodeEdge]
+    node_inputs: List[NodeInput]
+
+
+NodeTypeLiteral = Literal['input', 'output', 'process']
+TransformTypeLiteral = Literal['narrow', 'wide', 'other']
+
+
+class NodeDefault(BaseModel):
+    node_name: str
+    node_type: NodeTypeLiteral
+    transform_type: TransformTypeLiteral
+    has_default_settings: Optional[Any] = None