simstack 0.2.0__py3-none-any.whl

simstack/util/sample_config.py

@@ -0,0 +1,98 @@
+_SAMPLE_SIMSTACK_TOML_DICT = {
+    "parameters": {
+        "common": {
+            "resources": [
+                "local",
+                "int-nano",
+                "horeka",
+                "justus",
+                "self",
+                "exchange",
+                "uploads",
+            ],
+            "database": "wolfgang_data",
+            "test_database": "wolfgang_test",
+            "connection_string": "mongodb://localhost:27017/",
+        },
+        "local": {
+            "ssh-key": "C:\\Users\\bj7610\\Documents\\etc\\.ssh\\surface11_openssh",
+            "resource": "local",
+            "workdir": "C:\\Users\\bj7610\\simstack",
+            "python_path": [
+                "C:\\Users\\bj7610\\PyCharmProjects\\simstack-model",
+                "C:\\Users\\bj7610\\PyCharmProjects\\simstack-model\\src",
+            ],
+        },
+        "self": {
+            "ssh-key": "C:\\Users\\bj7610\\Documents\\etc\\.ssh\\surface11_openssh",
+            "resource": "local",
+            "workdir": "C:\\Users\\bj7610\\simstack",
+            "python_path": [
+                "C:\\Users\\bj7610\\PyCharmProjects\\simstack-model",
+                "C:\\Users\\bj7610\\PyCharmProjects\\simstack-model\\src",
+            ],
+        },
+        "uploads": {
+            "ssh-key": "C:\\Users\\bj7610\\Documents\\etc\\.ssh\\surface11_openssh",
+            "resource": "self",
+            "workdir": "C:\\Users\\bj7610\\simstack",
+            "python_path": [
+                "C:\\Users\\bj7610\\PyCharmProjects\\simstack-model",
+                "C:\\Users\\bj7610\\PyCharmProjects\\simstack-model\\src",
+            ],
+        },
+        "int-nano": {
+            "ssh-key": "/home/ws/bj7610/.ssh/idrsa",
+            "workdir": "/home/ws/bj7610/simstack",
+            "python_path": [
+                "/home/ws/bj7610/projects/simstack-model",
+                "/home/ws/bj7610/projects/simstack-model/src",
+            ],
+            "environment_start": "conda activate simstack-env",
+        },
+    },
+    "server": {
+        "port": 8000,
+        "secret_key": "secret_key",
+        "upload_dir": "C:\\Users\\bj7610\\simstack\\uploads",
+    },
+    "hosts": {
+        "local": "localhost",
+        "int-nano": "int-nano.int.kit.edu",
+        "justus": "justus.int.kit.edu",
+        "horeka": "horeka.int.kit.edu",
+    },
+    "routes": [
+        {"source": "local", "target": "int-nano", "host": "local"},
+        {"source": "int-nano", "target": "local", "host": "local"},
+        {"source": "horeka", "target": "local", "host": "horeka"},
+        {"source": "local", "target": "horeka", "host": "horeka"},
+        {"source": "justus", "target": "local", "host": "justus"},
+        {"source": "local", "target": "justus", "host": "justus"},
+    ],
+    "paths": {
+        "models": {"path": "src/simstack/models", "drops": "src"},
+        "methods": {"path": "src/simstack/methods", "drops": "src"},
+        "tests": {"path": "tests", "drops": "", "use_pickle": True},
+    },
+}
+
+
+def sample_config_writer_entrypoint() -> None:
+    """
+    Entry point to write a sample simstack.toml configuration file.
+    """
+    import os
+    import toml
+
+    config_path = os.path.join(os.getcwd(), "simstack.toml")
+    if os.path.exists(config_path):
+        print(
+            f"simstack.toml already exists at {config_path}. Aborting to avoid overwrite."
+        )
+        return
+
+    with open(config_path, "w") as config_file:
+        toml.dump(_SAMPLE_SIMSTACK_TOML_DICT, config_file)
+
+    print(f"Sample simstack.toml written to {config_path}")

simstack/util/sensitive_content_filter.py

@@ -0,0 +1,48 @@
+from typing import Dict, Any
+
+
+def filter_file_content(data: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Recursively searches through nested data structures to filter out sensitive content.
+    Specifically looks for 'content' fields in dictionaries, especially within 'files' structures.
+
+    Args:
+        data: Any Python data structure (dict, list, etc.)
+
+    Returns:
+        The filtered data structure with sensitive content removed
+    """
+    if isinstance(data, dict):
+        # Create a new dict to avoid modifying the original during iteration
+        result = {}
+
+        # Check if this is a FileStack-like dict with content
+        if "content" in data:
+            # Make a copy of the dict without the content field
+            result = {
+                k: filter_file_content(v) for k, v in data.items() if k != "content"
+            }
+        else:
+            # Process all items recursively
+            for key, value in data.items():
+                result[key] = filter_file_content(value)
+
+        return result
+
+    elif isinstance(data, list):
+        # Process each item in the list recursively
+        return [filter_file_content(item) for item in data]
+
+    elif hasattr(data, "__dict__"):
+        # Handle object instances by converting to dict and processing
+        obj_dict = data.__dict__.copy()
+        filtered_dict = filter_file_content(obj_dict)
+
+        # This is tricky - we'd ideally create a new instance with the filtered data
+        # but that's complex without knowing the class structure
+        # So we'll return the filtered dict instead
+        return filtered_dict
+
+    else:
+        # For primitive types, return as is
+        return data

simstack/util/setup_logging.py

@@ -0,0 +1,49 @@
+import logging
+
+from simstack.util.db_logger import DBLogHandler
+
+
+def setup_logging(
+    connection_string: str,
+    db_name: str,
+    log_level=logging.INFO,
+    console=True,
+    log_format: str = "%(asctime)s - %(name)-15s - %(levelname)-10s - %(filename)-20s:%(lineno)4d - %(message)s",
+):
+    """
+    Configures the root logger with specified logging level, handlers for both database
+    and optionally the console.
+
+    :param log_format:
+    :param connection_string: A string representing the database connection.
+    :param db_name: A name of the database to log into.
+    :param log_level: Logging verbosity level, defaulting to logging.INFO.
+    :param console: A boolean indicating whether to log messages to the console.
+    :return: Configured root logger instance.
+    """
+    # Configure the root logger
+    root_logger = logging.getLogger()
+    root_logger.setLevel(log_level)
+
+    # Remove any existing handlers to avoid duplicates during reloads
+    for handler in root_logger.handlers[:]:
+        root_logger.removeHandler(handler)
+
+    # Create formatter
+    formatter = logging.Formatter(log_format, datefmt="%Y-%m-%d %H:%M:%S")
+
+    db_handler = DBLogHandler(connection_string, db_name, "logs")
+
+    # Add database handler
+    db_handler.setLevel(log_level)
+    db_handler.setFormatter(formatter)
+    root_logger.addHandler(db_handler)
+
+    # Optionally add console handler
+    if console:
+        console_handler = logging.StreamHandler()
+        console_handler.setFormatter(formatter)
+        console_handler.setLevel(log_level)
+        root_logger.addHandler(console_handler)
+
+    return root_logger

simstack/util/submit_to_watchdog.py

@@ -0,0 +1,106 @@
+# python
+import os
+import time
+from pathlib import Path
+from uuid import uuid4
+
+from pydantic import BaseModel
+
+POLL_INTERVAL = 1.0
+TIMEOUT_SEC = 60 * 60 * 72  # adjust
+
+
+class WatchdogResult(BaseModel):
+    job_id: str
+    status: str
+    returncode: int | None
+    stdout: str
+    stderr: str
+    paths: dict[str, str]
+
+
+def submit_to_watchdog(
+    command: str,
+    job_id: str,
+    queue_dir: Path = Path.cwd() / "queue",
+    env: dict | None = None,
+    cwd: str | None = None,
+) -> WatchdogResult:
+    """
+    Submit a command to be executed by the file-queue watcher.
+    Returns a WatchdogResult with status, exit_code, stdout, stderr, and paths.
+    """
+    base = queue_dir / job_id
+    cmd_file_tmp = base.with_suffix(".cmd.tmp")
+    cmd_file = base.with_suffix(".cmd")
+    ready_file = base.with_suffix(".ready")
+    done_file = base.with_suffix(".done")
+    fail_file = base.with_suffix(".fail")
+    exit_file = base.with_suffix(".exit")
+    out_file = base.with_suffix(".stdout")
+    err_file = base.with_suffix(".stderr")
+
+    spec_lines = []
+    if cwd:
+        spec_lines.append(f"CWD={cwd}")
+    if env:
+        for k, v in env.items():
+            spec_lines.append(f"ENV[{k}]={v}")
+    spec_lines.append("CMD=" + command)
+    data = "\n".join(spec_lines) + "\n"
+
+    queue_dir.mkdir(parents=True, exist_ok=True)
+    print(f"[watcher] Submitting {data} to {cmd_file}")
+    with open(cmd_file_tmp, "w", encoding="utf-8") as f:
+        f.write(data)
+        f.flush()
+        os.fsync(f.fileno())
+
+    os.replace(cmd_file_tmp, cmd_file)  # atomic
+    ready_file.touch()  # signal
+
+    deadline = time.time() + TIMEOUT_SEC
+    while time.time() < deadline:
+        if done_file.exists() or fail_file.exists():
+            break
+        time.sleep(POLL_INTERVAL)
+
+    result = WatchdogResult(
+        job_id=job_id,
+        status="timeout",
+        returncode=None,
+        stdout="",
+        stderr="",
+        paths={
+            "cmd": str(cmd_file),
+            "ready": str(ready_file),
+            "done": str(done_file),
+            "fail": str(fail_file),
+            "exit": str(exit_file),
+            "stdout": str(out_file),
+            "stderr": str(err_file),
+        },
+    )
+
+    if done_file.exists() or fail_file.exists():
+        exit_code: int | None = None
+        if exit_file.exists():
+            try:
+                exit_code = int(exit_file.read_text().strip())
+            except Exception:
+                exit_code = None
+        out = out_file.read_text(errors="replace") if out_file.exists() else ""
+        err = err_file.read_text(errors="replace") if err_file.exists() else ""
+        ok = done_file.exists() and (exit_code == 0)
+
+        result.status = "ok" if ok else "failed"
+        result.returncode = exit_code
+        result.stdout = out
+        result.stderr = err
+
+    return result
+
+
+if __name__ == "__main__":
+    res = submit_to_watchdog("dir", uuid4().hex)
+    print(res)

simstack/util/ui_tools.py

@@ -0,0 +1,213 @@
+from typing import Dict, Any, List
+
+
+def ui_hide_fields(ui_schema: Dict[str, Any], fields: str | List[str]) -> dict:
+    """
+    Hides specified fields in the JSON schema.
+
+    Args:
+        ui_schema: The original JSON schema
+        fields: List of field names to hide
+    """
+
+    local_fields = [fields] if isinstance(fields, str) else fields
+    # Iterate through the fields to hide
+    for field in local_fields:
+        ui_schema[field] = {"ui:widget": "hidden"}
+    return ui_schema
+
+
+def ui_make_properties_optional(
+    json_schema: Dict[str, Any], properties: List[str], option_field: str = "Show More"
+) -> dict:
+    """
+    Modifies a JSON schema to make specified properties visible only when an option is enabled.
+
+    Args:
+        cls: The class to which the JSON schema belongs
+        json_schema: The original JSON schema
+        properties: List of property names to make optional
+        option_field: Name of the boolean field that controls visibility
+
+    Returns:
+        Modified JSON schema with dependencies
+    """
+    # Add the option field
+
+    if option_field in json_schema["properties"]:
+        assert json_schema["properties"][option_field]["type"] == "boolean"
+    else:
+        json_schema["properties"][option_field] = {
+            "type": "boolean",
+            "title": option_field,
+            "default": False,
+        }
+
+    # Create dependencies object if it doesn't exist
+    if "dependencies" not in json_schema:
+        json_schema["dependencies"] = {}
+
+    new_props = {}
+    for prop in properties:
+        if prop not in json_schema["properties"]:
+            raise ValueError(f"Property {prop} is not in json_schema")
+        new_props[prop] = json_schema["properties"][prop]
+        del json_schema["properties"][prop]
+
+    # Add dependency rules for each property
+    sub_schema = {"properties": {option_field: {"enum": [True]}, **new_props}}
+    json_schema["dependencies"][option_field] = {
+        "oneOf": [
+            {
+                "properties": {
+                    option_field: {"enum": [False]},
+                    **{prop: {"type": "null"} for prop in properties},
+                }
+            },
+            sub_schema,
+        ]
+    }
+
+    return sub_schema
+
+
+#
+# def ui_make_dependencies_optional(cls, json_schema: Dict[str, Any], option_field_path: List[str],
+#                                   properties: List[str], new_option_field: str = "Show More Details") -> Dict[str, Any]:
+#     """
+#     Makes properties optional within an already optional section at any nesting level.
+#
+#     Args:
+#         cls: The class to which the JSON schema belongs
+#         json_schema: The original JSON schema
+#         option_field_path: List of option field names representing the path to the target subschema
+#         properties: List of property names to make optional within the subschema
+#         new_option_field: Name of the new boolean field that controls visibility
+#
+#     Returns:
+#         Modified JSON schema with nested optional properties
+#     """
+#
+#     # Find the target subschema by traversing the path
+#     def find_subschema(schema, path, current_depth=0):
+#         if current_depth >= len(path):
+#             return schema
+#
+#         current_field = path[current_depth]
+#
+#         if "dependencies" not in schema or current_field not in schema["dependencies"]:
+#             raise ValueError(f"Option field '{current_field}' not found in dependencies at depth {current_depth}")
+#
+#         dependency = schema["dependencies"][current_field]
+#         if "oneOf" not in dependency or len(dependency["oneOf"]) < 2:
+#             raise ValueError(f"Invalid dependency structure for '{current_field}' at depth {current_depth}")
+#
+#         # The subschema is the second element in the oneOf array (index 1)
+#         subschema = dependency["oneOf"][1]
+#
+#         # Continue traversing
+#         return find_subschema(subschema, path, current_depth + 1)
+#
+#     # Find the target subschema
+#     target_schema = find_subschema(json_schema, option_field_path)
+#
+#     # Add the new option field to the target schema
+#     target_schema["properties"][new_option_field] = {
+#         "type": "boolean",
+#         "title": new_option_field,
+#         "default": False
+#     }
+#
+#     # Move specified properties to a new dependency within the target schema
+#     new_props = {}
+#     for prop in properties:
+#         if prop not in target_schema["properties"]:
+#             raise ValueError(f"Property '{prop}' is not in the target schema")
+#         new_props[prop] = target_schema["properties"][prop]
+#         del target_schema["properties"][prop]
+#
+#     # Create dependencies in the target schema if it doesn't exist
+#     if "dependencies" not in target_schema:
+#         target_schema["dependencies"] = {}
+#
+#     # Add nested dependency
+#     nested_sub_schema = {
+#         "properties": {
+#             new_option_field: {"enum": [True]},
+#             **new_props
+#         }
+#     }
+#
+#     target_schema["dependencies"][new_option_field] = {
+#         "oneOf": [
+#             {
+#                 "properties": {
+#                     new_option_field: {"enum": [False]},
+#                     **{prop: {"type": "null"} for prop in properties}
+#                 }
+#             },
+#             nested_sub_schema
+#         ]
+#     }
+#
+#     # The changes are made directly to the target schema which is a reference
+#     # to part of json_schema, so we don't need to explicitly update json_schema
+#     return json_schema
+#
+
+
+def ui_make_title(cls, ui_schema: Dict[str, Any], field: str, title: str) -> dict:
+    """
+    Adds a title to the JSON schema.
+
+    Args:
+        cls: The class to which the JSON schema belongs
+        ui_schema: The original ui_schema schema
+        title: Title to be added
+
+    Returns:
+        Modified JSON schema with title
+    """
+    ui_schema[field] = {"ui:title": title}
+    return ui_schema
+
+
+def ui_make_foldable(ui_schema: Dict[str, Any], field: str) -> dict:
+    """
+    Adds a foldable section to the JSON schema.
+
+    Args:
+        field: The field to be made foldable
+        ui_schema: The original ui_schema schema
+    Returns:
+        Modified ui_schema with foldable section
+
+    """
+    # TODO add checks that the fields actually exist
+    if field not in ui_schema:
+        ui_schema[field] = {}
+    if "ui:options" not in ui_schema:
+        ui_schema[field]["ui:options"] = {}
+    ui_schema[field]["ui:options"] = {
+        "ui:foldable": "true",
+    }
+    return ui_schema
+
+
+def ui_line_vector(ui_schema: Dict[str, Any], field: str) -> Dict[str, Any]:
+    """
+    Modifies the JSON schema to represent a line vector.
+
+    Args:
+        ui_schema: The original JSON schema
+
+    Returns:
+        Modified JSON schema with line vector representation
+    """
+    # TODO add checks that the fields actually exist and is a list
+    if field not in ui_schema:
+        ui_schema[field] = {}
+    if "ui:options" not in ui_schema:
+        ui_schema[field]["ui:options"] = {}
+    ui_schema[field]["ui:options"]["ui:field"] = "LineVector"
+    return ui_schema