setta 0.0.1.dev0__py3-none-any.whl → 0.0.2.dev0__py3-none-any.whl

setta/database/db/uiTypes/save.py

@@ -0,0 +1,99 @@
+import json
+
+from setta.utils.constants import DEFAULT_VALUES, UI_TYPE_TABLE_DATA_JSON_FIELDS
+from setta.utils.utils import filter_dict
+
+
+def save_ui_types(db, uiTypes):
+    query = """
+        INSERT INTO UIType (id, type, data)
+        VALUES (:id, :type, :data)
+        ON CONFLICT (id)
+        DO UPDATE SET
+            type = :type,
+            data = :data
+    """
+
+    query_parameters = [
+        {
+            "id": v["id"],
+            "type": v["type"],
+            "data": json.dumps(
+                filter_dict(v, UI_TYPE_TABLE_DATA_JSON_FIELDS, DEFAULT_VALUES["uiType"])
+            ),
+        }
+        for v in uiTypes.values()
+    ]
+    db.executemany(query, query_parameters)
+
+
+def save_ui_type_cols(db, uitype_cols):
+    query = """
+        INSERT INTO UITypeColId (id)
+        VALUES (:id)
+        ON CONFLICT (id)
+        DO NOTHING
+    """
+
+    query_params = [{"id": k} for k in uitype_cols.keys()]
+    db.executemany(query, query_params)
+
+    query_params = []
+    for idid in uitype_cols.keys():
+        for code_info_id in uitype_cols[idid].keys():
+            query_params.extend([idid, code_info_id])
+    placeholders = ", ".join(["(?, ?)"] * (len(query_params) // 2))
+    query = f"""
+        UPDATE UIType
+        SET asNonPresetForUITypeColId = null,
+            asNonPresetForCodeInfoId = null
+        WHERE (asNonPresetForUITypeColId, asNonPresetForCodeInfoId) IN ({placeholders})
+    """
+    db.execute(query, query_params)
+
+    placeholders = ", ".join(["?"] * len(uitype_cols))
+    query = f"""
+        DELETE FROM UITypeCol
+        WHERE idid IN ({placeholders})
+    """
+    query_params = list(uitype_cols.keys())
+    db.execute(query, query_params)
+
+    query = """
+        INSERT INTO UITypeCol (idid, codeInfoId, uiTypeId)
+        VALUES (:idid, :codeInfoId, :uiTypeId)
+    """
+    query_params = []
+    for idid in uitype_cols.keys():
+        for code_info_id, ui_info in uitype_cols[idid].items():
+            query_params.append(
+                {
+                    "idid": idid,
+                    "codeInfoId": code_info_id,
+                    "uiTypeId": ui_info["uiTypeId"],
+                }
+            )
+
+    db.executemany(query, query_params)
+
+    query = """
+        UPDATE UIType
+        SET asNonPresetForUITypeColId = :uiTypeColId,
+            asNonPresetForCodeInfoId = :codeInfoId
+        WHERE id = :nonPresetUITypeId
+    """
+    query_params = []
+    for idid in uitype_cols.keys():
+        for code_info_id, ui_info in uitype_cols[idid].items():
+            query_params.extend(
+                [
+                    {
+                        "uiTypeColId": idid,
+                        "codeInfoId": code_info_id,
+                        "nonPresetUITypeId": x,
+                    }
+                    for x in ui_info["nonPresetUITypeIds"]
+                ]
+            )
+
+    db.executemany(query, query_params)

setta/database/db/uiTypes/utils.py

@@ -0,0 +1,27 @@
+import copy
+
+from setta.utils.constants import DEFAULT_VALUES
+from setta.utils.utils import recursive_dict_merge
+
+
+def add_defaults_to_ui_types(ui_types):
+    for id in ui_types.keys():
+        ui_types[id] = with_ui_type_defaults(**ui_types[id])
+
+
+def with_ui_type_defaults(**kwargs):
+    return recursive_dict_merge(copy.deepcopy(DEFAULT_VALUES["uiType"]), kwargs)
+
+
+def new_ui_type_col_entry():
+    return copy.deepcopy(DEFAULT_VALUES["uiTypeColEntry"])
+
+
+def add_defaults_to_ui_type_cols(ui_type_cols):
+    for v in ui_type_cols.values():
+        for paramId in v.keys():
+            v[paramId] = with_ui_type_col_entry_defaults(**v[paramId])
+
+
+def with_ui_type_col_entry_defaults(**kwargs):
+    return recursive_dict_merge(copy.deepcopy(DEFAULT_VALUES["uiTypeColEntry"]), kwargs)

setta/database/db_init.py

@@ -0,0 +1,36 @@
+from setta.utils.constants import CONSTANTS_FOLDER
+from setta.utils.utils import get_absolute_path
+
+from .seed import seed
+
+
+def create_tables(db):
+    with open(
+        get_absolute_path(__file__, CONSTANTS_FOLDER / "db_init.sql"), "r"
+    ) as sql_file:
+        sql_script = sql_file.read()
+        db.executescript(sql_script)
+
+
+def seed_data_exist(db, tablenames):
+    for table in tablenames:
+        db.execute(f"SELECT COUNT(*) FROM {table}")
+        if db.fetchone()[0] == 0:
+            return False
+    return True
+
+
+def maybe_create_tables_and_seed(dbq, with_examples=False, with_base_ui_types=True):
+    dbq.connect()
+    with dbq as db:
+        create_tables(db)
+        do_seed_examples = with_examples and not seed_data_exist(db, ["ProjectConfig"])
+        do_seed_base_ui_types = with_base_ui_types and not seed_data_exist(
+            db, ["UIType"]
+        )
+        seed(
+            db,
+            with_examples=do_seed_examples,
+            with_base_ui_types=do_seed_base_ui_types,
+        )
+    dbq.disconnect()

setta/database/db_objs.py

@@ -0,0 +1,102 @@
+import queue
+import sqlite3
+
+
+class DB:
+    def __init__(self, path):
+        self.conn = None
+        self.cursor = None
+        self.path = path
+
+    def connect(self):
+        if not self.conn:
+            self.conn = sqlite3.connect(
+                self.path,
+                check_same_thread=False,
+            )
+            self.conn.row_factory = sqlite3.Row
+            self.cursor = self.conn.cursor()
+        self.execute("PRAGMA foreign_keys = ON;")
+
+    def disconnect(self):
+        if self.conn:
+            self.cursor.close()
+            self.conn.close()
+            self.cursor = None
+            self.conn = None
+
+    def commit(self):
+        self.conn.commit()
+
+    def rollback(self):
+        self.conn.rollback()
+
+    def execute(self, *args, **kwargs):
+        self.cursor.execute(*args, **kwargs)
+
+    def executemany(self, *args, **kwargs):
+        self.cursor.executemany(*args, **kwargs)
+
+    def executescript(self, *args, **kwargs):
+        self.cursor.executescript(*args, **kwargs)
+
+    def fetchall(self, *args, **kwargs):
+        return self.cursor.fetchall(*args, **kwargs)
+
+    def fetchone(self, *args, **kwargs):
+        return self.cursor.fetchone(*args, **kwargs)
+
+    @property
+    def description(self):
+        return self.cursor.description
+
+    def __enter__(self, *args, **kwargs):
+        self.conn.__enter__(*args, **kwargs)
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        return self.conn.__exit__(*args, **kwargs)
+
+
+class DBQueue:
+    def __init__(self, path, n=1):
+        self.queue = queue.Queue()
+        for _ in range(n):
+            self.queue.put(DB(path))
+        self.curr = None
+
+    def set_curr(self):
+        self.curr = self.queue.get()
+
+    def put_back_curr(self):
+        self.queue.put(self.curr)
+        self.curr = None
+
+    def process_conns(self, fn_name):
+        all_dbs = []
+        while True:
+            try:
+                db = self.queue.get_nowait()
+            except queue.Empty:
+                break
+            else:
+                fn = getattr(db, fn_name)
+                fn()
+                all_dbs.append(db)
+
+        for db in all_dbs:
+            self.queue.put(db)
+
+    def connect(self):
+        self.process_conns("connect")
+
+    def disconnect(self):
+        self.process_conns("disconnect")
+
+    def __enter__(self, *args, **kwargs):
+        self.set_curr()
+        return self.curr.__enter__(*args, **kwargs)
+
+    def __exit__(self, *args, **kwargs):
+        self.curr.__exit__(*args, **kwargs)
+        self.put_back_curr()

setta/database/db_path.py

@@ -0,0 +1,8 @@
+import os
+
+DEFAULT_DB_FOLDER = os.getcwd()
+DEFAULT_DB_NAME = "setta.db"
+
+
+def get_default_db_path():
+    return os.path.join(DEFAULT_DB_FOLDER, DEFAULT_DB_NAME)

setta/database/export_db/export_db.py

@@ -0,0 +1,43 @@
+import os
+
+from setta.utils.constants import USER_SETTINGS
+
+from ..db_objs import DBQueue
+from .export_raw import export_raw
+from .export_readable import export_readable
+
+
+def _export_database(
+    db, filename, raw=True, readable=True, readable_with_variants=True
+):
+    if not raw and not readable and not readable_with_variants:
+        return
+    filename = os.path.splitext(os.path.basename(filename))[0]
+    if raw:
+        export_raw(db, filename)
+    if readable:
+        export_readable(db, filename, with_variants=False)
+    if readable_with_variants:
+        export_readable(db, filename, with_variants=True)
+
+
+def export_database(path):
+    # Connect to the SQLite database
+    dbq = DBQueue(path)
+    dbq.connect()
+    with dbq as db:
+        # Export the database, including schema and data, to JSON and YAML
+        _export_database(db, path, True, True, True)
+    dbq.disconnect()
+
+
+def maybe_export_database(db, path):
+    _export_database(
+        db,
+        path,
+        raw=USER_SETTINGS["backend"]["exportDbRawOnSave"],
+        readable=USER_SETTINGS["backend"]["exportDbReadableOnSave"],
+        readable_with_variants=USER_SETTINGS["backend"][
+            "exportDbReadableWithVariantsOnSave"
+        ],
+    )

setta/database/export_db/export_raw.py

@@ -0,0 +1,53 @@
+import logging
+import os
+from pathlib import Path
+
+import yaml
+
+logger = logging.getLogger(__name__)
+
+
+def export_raw(db, filename):
+    schema = fetch_schema(db)
+    data = fetch_all_data(db)
+
+    # Combined schema and data for JSON export
+    database_export = {"schema": schema, "data": data}
+
+    # with open(f"{filename}_export.json", 'w') as file:
+    #     json.dump(database_export, file, indent=4)
+
+    # Customize the YAML dump to use literal style for SQL
+    def literal_presenter(dumper, data):
+        if "\n" in data:
+            return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|")
+        return dumper.represent_scalar("tag:yaml.org,2002:str", data)
+
+    yaml.add_representer(str, literal_presenter)
+
+    # For YAML export, PyYAML is used; ensure it's installed
+    filepath = Path(os.getcwd()) / f"{filename}_export.yaml"
+    logger.debug(f"saving raw db to {filepath}")
+    with open(filepath, "w") as file:
+        yaml.dump(database_export, file, allow_unicode=True, sort_keys=False)
+
+
+def fetch_schema(cursor):
+    """Fetch the schema of all tables in the database."""
+    cursor.execute("SELECT type, name, sql FROM sqlite_master WHERE sql NOT NULL;")
+    schema = {}
+    for type, name, sql in cursor.fetchall():
+        if name not in schema:
+            schema[name] = {"type": type, "sql": sql}
+    return schema
+
+
+def fetch_all_data(cursor):
+    """Fetch all data from the database."""
+    data = {}
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+    for (table_name,) in cursor.fetchall():
+        cursor.execute(f"SELECT * FROM {table_name}")
+        columns = [description[0] for description in cursor.description]
+        data[table_name] = [dict(zip(columns, row)) for row in cursor.fetchall()]
+    return data

setta/database/export_db/export_readable.py

@@ -0,0 +1,242 @@
+import logging
+import os
+import shutil
+from pathlib import Path
+
+from setta.cli.logger import save_base64_to_png
+from setta.code_gen.export_selected import (
+    export_param_sweep_section_to_dict,
+    export_section_to_dict,
+    export_single_run_group_to_dict,
+    get_for_section_id,
+    get_section_name,
+    get_section_type,
+    get_section_variant_attr,
+    get_section_variant_ids,
+    get_selected_section_variant,
+    get_selected_section_variant_id,
+    get_variant_code,
+    get_variant_name,
+)
+from setta.code_gen.yaml.section_dict import (
+    param_sweep_dict_to_yaml,
+    run_group_dict_to_yaml,
+    section_dict_to_yaml,
+)
+from setta.database.db.artifacts.load import (
+    load_artifact_metadata_and_maybe_value_from_disk,
+)
+from setta.utils.constants import C
+from setta.utils.section_contents import getParamIdsToShowInArea
+
+from .utils import get_configs_and_root_folder
+
+logger = logging.getLogger(__name__)
+
+
+def export_readable(db, filename, with_variants):
+    configs, root_folder = get_configs_and_root_folder(db, filename)
+    subfolder_name = "with_variants" if with_variants else "clean"
+    root_folder = root_folder / subfolder_name
+    try:
+        logger.debug(f"try deleting: {root_folder}")
+        shutil.rmtree(root_folder)
+    except:
+        pass
+
+    logger.debug(f"exporting readable to {root_folder}")
+
+    artifactIds = set()
+    for c in configs:
+        for ag in c["artifactGroups"].values():
+            for t in ag["artifactTransforms"]:
+                artifactIds.add(t["artifactId"])
+
+    artifacts = load_artifact_metadata_and_maybe_value_from_disk(db, list(artifactIds))
+
+    for c in configs:
+        c["artifacts"] = artifacts
+        export_config(root_folder, c, with_variants)
+
+
+def export_config(folder, config, with_variants):
+    config_folder = Path(folder) / config["projectConfig"]["name"]
+    for k in config["projectConfig"]["children"].keys():
+        os.makedirs(config_folder, exist_ok=True)
+        export_section(config_folder, config, k, with_variants)
+
+
+def export_section(folder, config, section_id, with_variants):
+    exporter_fn = get_exporter_fn(config, section_id)
+    if not exporter_fn:
+        return
+    variant_ids = (
+        get_section_variant_ids(config, section_id)
+        if with_variants
+        else [get_selected_section_variant_id(config, section_id)]
+    )
+    for variant_id in variant_ids:
+        if not variant_id:
+            continue
+        exporter_fn(folder, config, section_id, variant_id, with_variants)
+
+
+def export_code(folder, config, section_id, variant_id, with_variants):
+    codeLanguage = config["sections"][section_id]["codeLanguage"]
+    code = get_variant_code(config, variant_id)
+    base = maybe_with_variant_in_filename(config, section_id, variant_id, with_variants)
+    extension = {"javascript": "js", "python": "py", "bash": "sh"}[codeLanguage]
+    with open(f"{folder / base}.{extension}", "w") as file:
+        file.write(code)
+
+
+def export_regular_section(folder, config, section_id, variant_id, with_variants):
+    sectionVariant = config["sectionVariants"][variant_id]
+    selectedItemId = sectionVariant["selectedItem"]
+    codeInfoChildren = (
+        config["codeInfoCols"]
+        .get(sectionVariant["codeInfoColId"], {})
+        .get("children", {})
+    )
+    pinned = getParamIdsToShowInArea(
+        True,
+        config["codeInfo"],
+        codeInfoChildren,
+        selectedItemId,
+    )
+    unpinned = getParamIdsToShowInArea(
+        False,
+        config["codeInfo"],
+        codeInfoChildren,
+        selectedItemId,
+    )
+    pinned["requiredParamIds"] = pinned["requiredParamIdsToPaths"].keys()
+    unpinned["requiredParamIds"] = unpinned["requiredParamIdsToPaths"].keys()
+    del pinned["requiredParamIdsToPaths"]
+    del unpinned["requiredParamIdsToPaths"]
+
+    output = export_section_to_dict(
+        config["codeInfo"],
+        codeInfoChildren,
+        sectionVariant,
+        {"pinned": pinned, "unpinned": unpinned},
+    )
+    yamlValue = section_dict_to_yaml(output, config["codeInfo"], sectionVariant)
+    base = maybe_with_variant_in_filename(config, section_id, variant_id, with_variants)
+
+    with open(folder / f"{base}.yaml", "w") as file:
+        file.write(yamlValue)
+
+
+def export_list_section(folder, config, section_id, variant_id, with_variants):
+    subfolder_name = maybe_with_variant_in_filename(
+        config, section_id, variant_id, with_variants
+    )
+    children_folder = folder / subfolder_name
+    os.makedirs(children_folder, exist_ok=True)
+    for c in config["sectionVariants"][variant_id]["children"]:
+        export_section(children_folder, config, c, with_variants)
+
+
+def export_info_section(folder, config, section_id, variant_id, with_variants):
+    base = maybe_with_variant_in_filename(config, section_id, variant_id, with_variants)
+    section = config["sections"][section_id]
+    description = get_section_variant_attr(config, variant_id, "description")
+    renderMarkdown = section["renderMarkdown"]
+    extension = ".md" if renderMarkdown else ".txt"
+    with open(folder / f"{base}{extension}", "w") as file:
+        file.write(description)
+
+
+def export_param_sweep_section(folder, config, section_id, variant_id, with_variants):
+    for_section_id = get_for_section_id(config, section_id)
+    if not for_section_id:
+        return
+
+    sectionVariant = get_selected_section_variant(config, for_section_id)
+    codeInfoChildren = config["codeInfoCols"][sectionVariant["codeInfoColId"]][
+        "children"
+    ]
+    sweep = config["sectionVariants"][variant_id]["sweep"]
+    e = export_param_sweep_section_to_dict(sweep, config["codeInfo"], codeInfoChildren)
+    yamlValue = param_sweep_dict_to_yaml(e)
+    base = maybe_with_variant_in_filename(config, section_id, variant_id, with_variants)
+    with open(folder / f"{base}.yaml", "w") as file:
+        file.write(yamlValue)
+
+
+def export_global_param_sweep_section(folder, config, section_id, *_, **__):
+    subfolder_name = get_section_name(config, section_id)
+    folder = folder / subfolder_name
+    os.makedirs(folder, exist_ok=True)
+    variantIds = config["sections"][section_id]["variantIds"]
+    for v in variantIds:
+        variant = config["sectionVariants"][v]
+        run_group = variant["runGroup"]
+        e = export_single_run_group_to_dict(
+            run_group,
+            config["sections"],
+            config["sectionVariants"],
+        )
+        yamlValue = run_group_dict_to_yaml(e)
+        base = variant["name"]
+        with open(folder / f"{base}.yaml", "w") as file:
+            file.write(yamlValue)
+
+
+def export_social_section(folder, config, section_id, *_, **__):
+    section = config["sections"][section_id]
+    url = section["social"]
+    base = get_section_name(config, section_id)
+    with open(folder / f"{base}.txt", "w") as file:
+        file.write(url)
+
+
+def export_image_section(folder, config, section_id, *_, **__):
+    section = config["sections"][section_id]
+    artifact_group_id = section["artifactGroupIds"][0]
+    artifactId = config["artifactGroups"][artifact_group_id]["artifactTransforms"][0][
+        "artifactId"
+    ]
+    artifact = config["artifacts"][artifactId]
+    base = get_section_name(config, section_id)
+    save_base64_to_png(artifact["value"], folder / f"{base}.png")
+
+
+def export_canvas_rendered_value(folder, config, section_id, *_, **__):
+    section = config["sections"][section_id]
+    if section["renderedValue"]:
+        base = get_section_name(config, section_id)
+        save_base64_to_png(section["renderedValue"], folder / f"{base}.png")
+
+
+def maybe_with_variant_in_filename(config, section_id, variant_id, with_variants):
+    section_name = get_section_name(config, section_id)
+    variant_name = get_variant_name(config, variant_id)
+    if with_variants:
+        return f"{section_name}@{variant_name}"
+    return section_name
+
+
+def get_exporter_fn(config, section_id):
+    stype = get_section_type(config, section_id)
+    if stype == C.CODE:
+        return export_code
+    elif stype in [C.SECTION, C.GLOBAL_VARIABLES]:
+        return export_regular_section
+    elif stype in [C.LIST_ROOT, C.DICT_ROOT, C.GROUP]:
+        return export_list_section
+    elif stype == C.INFO:
+        return export_info_section
+    elif stype == C.PARAM_SWEEP:
+        return export_param_sweep_section
+    elif stype == C.GLOBAL_PARAM_SWEEP:
+        return export_global_param_sweep_section
+    elif stype == C.DRAW:
+        return export_canvas_rendered_value
+    elif stype == C.IMAGE:
+        return export_image_section
+    elif stype == C.SOCIAL:
+        return export_social_section
+    elif stype == C.CHART:
+        return export_canvas_rendered_value

setta/database/export_db/utils.py

@@ -0,0 +1,16 @@
+import os
+from collections import defaultdict
+from pathlib import Path
+
+import yaml
+from yaml.representer import Representer
+
+from setta.database.db.projects.load import load_full_project
+
+
+def get_configs_and_root_folder(db, filename):
+    # export defaultdict the same way as dict
+    yaml.add_representer(defaultdict, Representer.represent_dict)
+    configs = load_full_project(db)
+    root_folder = Path(os.getcwd()) / f"{filename}_export"
+    return configs, root_folder

setta/database/import_db.py

@@ -0,0 +1,28 @@
+import sqlite3
+
+import yaml
+
+
+def import_database(yaml_filepath, new_database_path):
+    with open(yaml_filepath, "r") as file:
+        database_content = yaml.safe_load(file)
+
+    conn = sqlite3.connect(new_database_path)
+    cursor = conn.cursor()
+
+    for table_name, table_info in database_content["schema"].items():
+        sql = table_info["sql"]
+        cursor.execute(sql)
+
+    for table_name, rows in database_content["data"].items():
+        if not rows:
+            continue
+        # Quote column names to handle reserved keywords
+        columns = ['"' + column + '"' for column in rows[0].keys()]
+        placeholders = ", ".join(["?"] * len(columns))
+        sql = f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})"
+        for row in rows:
+            cursor.execute(sql, list(row.values()))
+
+    conn.commit()
+    conn.close()