setta 0.0.9.dev0__py3-none-any.whl → 0.0.9.dev1__py3-none-any.whl

setta/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.9.dev0"
+__version__ = "0.0.9.dev1"

setta/code_gen/python/generate_code.py

@@ -258,6 +258,8 @@ def convert_var_names_to_readable_form(
         # only create nice_var_name if we haven't already created it for this var_name
         if var_name not in var_name_to_nice_var_name_mapping:
             nice_name = exporter_obj.output[var_name]["name"]
+            # TODO: do the same thing on the frontend to find naming conflicts
+            nice_name = nice_name.replace(" ", "_")
             nice_var_name = create_nice_var_name(top_level_symbols, nice_name)
             top_level_symbols.add(nice_var_name)
             var_name_to_nice_var_name_mapping[var_name] = nice_var_name

setta/database/db/codeInfo/copy.py

@@ -1,11 +1,8 @@
 from setta.database.utils import remap_ids, rename_keys
-from setta.utils.constants import is_from_json_source
 
 
 def copy_code_info(code_info):
-    ids_to_rename = [k for k in code_info.keys() if not is_from_json_source(k)]
-    new_code_info, code_info_id_map = remap_ids(code_info, ids_to_rename)
-    code_info_id_map.update({k: k for k in code_info.keys() if is_from_json_source(k)})
+    new_code_info, code_info_id_map = remap_ids(code_info)
 
     for c in new_code_info.values():
         c["id"] = code_info_id_map[c["id"]]

setta/database/db/projects/saveAs.py

@@ -1,12 +1,10 @@
 import copy
 
-from ..sections.jsonSource import remove_json_source_data
 from .copy import copy_project_config, copy_project_details
 from .save import save_project_details
 
 
 def save_as_new_project_config(db, project, new_config_name, with_refs):
-    remove_json_source_data(project)
     if not with_refs:
         project_to_save = copy_project_details(
             project, new_config_name, do_create_new_id=True
@@ -21,7 +19,6 @@ def save_as_new_project_config(db, project, new_config_name, with_refs):
 
 
 def save_as_existing_project_config(db, project, config_name):
-    remove_json_source_data(project)
     query = """
         SELECT *
         FROM ProjectConfig

setta/database/db/projects/utils.py

@@ -43,8 +43,8 @@ def remove_empty(x):
     return {k: v for k, v in x.items() if len(v) > 0}
 
 
-def filter_data_for_json_export(p, keepCodeInfoThatHaveUITypes):
-    remove_json_source_data(p, keepCodeInfoThatHaveUITypes)
+def filter_data_for_json_export(p):
+    remove_json_source_data(p)
 
     p["projectConfig"] = filter_dict(
         p["projectConfig"],

setta/database/db/sectionVariants/copy.py

@@ -2,11 +2,20 @@ from setta.database.utils import remap_ids, rename_keys
 from setta.utils.generate_memorable_string import generate_memorable_string
 
 
-def copy_section_variants(section_variants, code_info_id_map, code_info_col_id_map):
+def copy_section_variants(
+    sections, section_variants, code_info_id_map, code_info_col_id_map
+):
     new_section_variants, section_variant_id_map = remap_ids(section_variants)
+    keep_old_name = set()
+    for s in sections.values():
+        if not s["jsonSource"]:
+            continue
+        for v in s["variantIds"]:
+            keep_old_name.add(section_variant_id_map[v])
 
-    for obj in new_section_variants.values():
-        obj["name"] = generate_memorable_string()
+    for id, obj in new_section_variants.items():
+        if id not in keep_old_name:
+            obj["name"] = generate_memorable_string()
         obj["values"] = rename_keys(obj["values"], code_info_id_map)
         if obj["codeInfoColId"]:
             obj["codeInfoColId"] = code_info_col_id_map[obj["codeInfoColId"]]

setta/database/db/sections/copy.py

@@ -18,6 +18,9 @@ def copy_sections(
     for section in new_sections.values():
         section["id"] = section_id_map[section["id"]]
         section["variantId"] = section_variant_id_map[section["variantId"]]
+        section["defaultVariantId"] = section_variant_id_map[
+            section["defaultVariantId"]
+        ]
         section["uiTypeId"] = ui_type_id_map[section["uiTypeId"]]
         if section["uiTypeColId"]:
             section["uiTypeColId"] = ui_type_col_id_map[section["uiTypeColId"]]
@@ -41,7 +44,7 @@ def copy_sections_and_other_info(x):
         x["codeInfoCols"], code_info_id_map
     )
     new_section_variants, section_variant_id_map = copy_section_variants(
-        x["sectionVariants"], code_info_id_map, code_info_col_id_map
+        x["sections"], x["sectionVariants"], code_info_id_map, code_info_col_id_map
     )
     new_ui_types, ui_type_id_map = copy_ui_types(x["uiTypes"])
     new_ui_type_cols, ui_type_col_id_map = copy_ui_type_cols(

setta/database/db/sections/jsonSource.py

@@ -1,8 +1,13 @@
 import json
 from collections import defaultdict
 
-from setta.utils.constants import BASE_UI_TYPE_IDS, C, is_from_json_source
-from setta.utils.utils import recursive_dict_merge, save_json_to_file, try_json
+from setta.utils.constants import BASE_UI_TYPE_IDS, C
+from setta.utils.utils import (
+    recursive_dict_merge,
+    replace_null_keys_with_none,
+    save_json_to_file,
+    try_json,
+)
 
 
 def save_json_source_data(p, section_ids=None, forking_from=None):
@@ -19,24 +24,29 @@ def save_json_source_data(p, section_ids=None, forking_from=None):
         with open(forking_from, "r") as f:
             forking_from_data = json.load(f)
 
+    p["codeInfoCols"] = replace_null_keys_with_none(p["codeInfoCols"])
+
+    ancestor_paths = build_ancestor_paths(p["codeInfo"], p["codeInfoCols"])
+
     for s in sections.values():
+        if not s["jsonSource"] or s["jsonSourceMissing"]:
+            continue
+
         for variantId in s["variantIds"]:
             variant = p["sectionVariants"][variantId]
-            codeInfoCol = p["codeInfoCols"][variant["codeInfoColId"]]
+            codeInfoColId = variant["codeInfoColId"]
+            codeInfoCol = p["codeInfoCols"][codeInfoColId]
             filename = variant["name"]
-            for k, children in codeInfoCol["children"].items():
-                if is_from_json_source(k):
-                    metadata = json.loads(k.removeprefix(C.JSON_SOURCE_PREFIX))
-                    key_path = metadata["key"]
-                    value = try_getting_value(variant, k, children)
-
-                    current_dict = add_key_path_to_dict(
-                        to_be_saved[filename], key_path[:-1]
-                    )
 
-                    # Set the value at the final position
-                    if key_path:  # Only set if we have a path
-                        current_dict[key_path[-1]] = value
+            recursively_add_keys(
+                p,
+                variant,
+                codeInfoCol,
+                to_be_saved[filename],
+                None,
+                s["jsonSourceKeys"],
+                ancestor_paths,
+            )
 
             # Make sure the jsonSourceKeys are present.
             # (They might not be because they are completely empty)
@@ -50,12 +60,60 @@ def save_json_source_data(p, section_ids=None, forking_from=None):
     return to_be_saved
 
 
-def try_getting_value(variant, codeInfoId, codeInfoChildren):
-    if len(codeInfoChildren) == 0:
-        if codeInfoId in variant["values"]:
-            return try_json(variant["values"][codeInfoId]["value"])
-        return ""
-    return {}
+def build_ancestor_paths(codeInfo, codeInfoCols):
+    parent_map = {}
+    for col in codeInfoCols.values():
+        for parent_id, children in col["children"].items():
+            for child_id in children:
+                parent_map[(codeInfo[child_id]["jsonSource"], child_id)] = parent_id
+
+    ancestor_paths = {}
+    for node_id in codeInfo:
+        if node_id not in ancestor_paths:
+            path = []
+            current_id = node_id
+
+            # Traverse up to build the path
+            while current_id is not None:
+                if current_id in codeInfo:  # Skip if not a valid codeInfo node
+                    name = codeInfo[current_id]["name"]
+                    path.insert(0, name)
+
+                    # Get parent using the map
+                    parent_id = parent_map.get(
+                        (codeInfo[current_id]["jsonSource"], current_id)
+                    )
+                    current_id = parent_id
+                else:
+                    break
+
+            ancestor_paths[node_id] = path
+
+    return ancestor_paths
+
+
+def recursively_add_keys(
+    p, variant, codeInfoCol, input_dict, codeInfoId, jsonSourceKeys, ancestor_paths
+):
+    for k in codeInfoCol["children"][codeInfoId]:
+        children = codeInfoCol["children"][k]
+        json_source = p["codeInfo"][k].get("jsonSource")
+
+        if json_source:
+            # Get pre-computed key path
+            key_path = [*jsonSourceKeys, *ancestor_paths[k]]
+            value = try_getting_value(variant, k, children)
+
+            current_dict = add_key_path_to_dict(input_dict, key_path[:-1])
+
+            # Set the value at the final position
+            if key_path:  # Only set if we have a path
+                current_dict[key_path[-1]] = value
+
+        # Continue recursion regardless of whether this node has a jsonSource
+        recursively_add_keys(
+            p, variant, codeInfoCol, input_dict, k, jsonSourceKeys, ancestor_paths
+        )
 
 
 def add_key_path_to_dict(output, key_path):
@@ -67,36 +125,43 @@ def add_key_path_to_dict(output, key_path):
     return output
 
 
-def condition_keep_code_info(k, jsonCodeInfoWithUIType, keepCodeInfoThatHaveUITypes):
-    if keepCodeInfoThatHaveUITypes:
-        return k in jsonCodeInfoWithUIType or not is_from_json_source(k)
-    return not is_from_json_source(k)
+def try_getting_value(variant, codeInfoId, codeInfoChildren):
+    if len(codeInfoChildren) == 0:
+        if codeInfoId in variant["values"]:
+            return try_json(variant["values"][codeInfoId]["value"])
+        return ""
+    return {}
+
 
+def condition_keep_code_info(codeInfo, jsonCodeInfoWithUIType):
+    if not codeInfo:
+        return False
+    return not codeInfo["jsonSource"] or codeInfo["id"] in jsonCodeInfoWithUIType
 
-def remove_json_source_data(p, keepCodeInfoThatHaveUITypes=True):
+
+def remove_json_source_data(p):
     for variant in p["sectionVariants"].values():
         variant["values"] = {
-            k: v for k, v in variant["values"].items() if not is_from_json_source(k)
+            k: v
+            for k, v in variant["values"].items()
+            if not p["codeInfo"][k]["jsonSource"]
         }
 
     jsonCodeInfoWithUIType = set()
-    if keepCodeInfoThatHaveUITypes:
-        for uiTypeCol in p["uiTypeCols"].values():
-            for paramInfoId, uiTypeInfo in uiTypeCol.items():
-                # we want to know which json source params have an associated uiTypeId
-                # only if it's not the base TEXT type, since that's the default
-                if (
-                    is_from_json_source(paramInfoId)
-                    and uiTypeInfo["uiTypeId"] != BASE_UI_TYPE_IDS[C.TEXT]
-                ):
-                    jsonCodeInfoWithUIType.add(paramInfoId)
+    for uiTypeCol in p["uiTypeCols"].values():
+        for paramInfoId, uiTypeInfo in uiTypeCol.items():
+            # we want to know which json source params have an associated uiTypeId
+            # only if it's not the base TEXT type, since that's the default
+            if (
+                p["codeInfo"][paramInfoId]["jsonSource"]
+                and uiTypeInfo["uiTypeId"] != BASE_UI_TYPE_IDS[C.TEXT]
+            ):
+                jsonCodeInfoWithUIType.add(paramInfoId)
 
     p["codeInfo"] = {
         k: v
         for k, v in p["codeInfo"].items()
-        if condition_keep_code_info(
-            k, jsonCodeInfoWithUIType, keepCodeInfoThatHaveUITypes
-        )
+        if condition_keep_code_info(v, jsonCodeInfoWithUIType)
     }
 
     for codeInfoColId in p["codeInfoCols"].keys():
@@ -106,7 +171,8 @@ def remove_json_source_data(p, keepCodeInfoThatHaveUITypes=True):
             for k, v in codeInfoCol["children"].items()
             if k is None
             or condition_keep_code_info(
-                k, jsonCodeInfoWithUIType, keepCodeInfoThatHaveUITypes
+                p["codeInfo"].get(k),
+                jsonCodeInfoWithUIType,
             )
         }
         for id, children in codeInfoCol["children"].items():
@@ -114,6 +180,7 @@ def remove_json_source_data(p, keepCodeInfoThatHaveUITypes=True):
                 c
                 for c in children
                 if condition_keep_code_info(
-                    c, jsonCodeInfoWithUIType, keepCodeInfoThatHaveUITypes
+                    p["codeInfo"].get(c),
+                    jsonCodeInfoWithUIType,
                 )
             ]

setta/database/db/sections/load.py

@@ -7,10 +7,10 @@ from collections import defaultdict
 
 from setta.database.db.artifacts.load import load_artifact_groups
 from setta.database.db.codeInfo.utils import new_code_info_col, with_code_info_defaults
+from setta.database.db.sections.jsonSource import build_ancestor_paths
 from setta.database.db.sections.utils import with_section_defaults
 from setta.database.db.sectionVariants.utils import new_ev_entry, new_section_variant
 from setta.database.utils import create_new_id
-from setta.utils.constants import C
 
 from ..sectionVariants.load import load_section_variants
 from ..uiTypes.load import load_uitypecols, load_uitypes
@@ -214,34 +214,9 @@ def load_json_sources_into_data_structures(
             f'Attempting to read {s["jsonSource"]} with keys {s["jsonSourceKeys"]}'
         )
         new_data = load_json_source(s["jsonSource"], s["jsonSourceKeys"])
-        logger.debug(f"Loaded json: {new_data}")
-        for filename, data in new_data.items():
-            codeInfo.update(data["codeInfo"])
-            variantId = None
-            for vid in s["variantIds"]:
-                if sectionVariants[vid]["name"] == filename:
-                    variantId = vid
-                    break
-            if not variantId:
-                variantId, section_variant = new_section_variant(
-                    name=filename,
-                )
-                s["variantIds"].append(variantId)
-                sectionVariants[variantId] = section_variant
-
-            curr_section_variant = sectionVariants[variantId]
-            curr_section_variant["values"] = data["sectionVariantValues"]
-            codeInfoColId = curr_section_variant["codeInfoColId"]
-
-            if not codeInfoColId:
-                codeInfoColId = create_new_id()
-                curr_section_variant["codeInfoColId"] = codeInfoColId
-                codeInfoCols[codeInfoColId] = new_code_info_col()
-
-            codeInfoCols[codeInfoColId]["children"] = data["codeInfoColChildren"]
-
-            s["configLanguage"] = "json"
-            filenames_loaded.add(filename)
+        filenames_loaded.update(
+            merge_into_existing(new_data, s, sectionVariants, codeInfo, codeInfoCols)
+        )
 
     # delete variants that aren't associated with a loaded file
     to_delete = []
@@ -281,6 +256,74 @@ def load_json_sources_into_data_structures(
             s["defaultVariantId"] = s["variantId"]
 
 
+def merge_into_existing(new_data, section, sectionVariants, codeInfo, codeInfoCols):
+    filenames_loaded = set()
+    jsonSourceMetadata_to_id = {}
+    ancestor_paths = build_ancestor_paths(codeInfo, codeInfoCols)
+    for id, info in codeInfo.items():
+        jsonSourceMetadata_to_id[
+            createMetadataJsonString(info["jsonSource"], ancestor_paths[id])
+        ] = id
+
+    for filename, data in new_data.items():
+        replacements = {}
+        new_ancestor_paths = build_ancestor_paths(
+            data["codeInfo"], {None: {"children": data["codeInfoColChildren"]}}
+        )
+        for newId, newInfo in data["codeInfo"].items():
+            existingId = jsonSourceMetadata_to_id.get(
+                createMetadataJsonString(
+                    newInfo["jsonSource"], new_ancestor_paths[newId]
+                )
+            )
+            if existingId:
+                replacements[newId] = existingId
+            else:
+                codeInfo[newId] = newInfo
+
+        for newId, existingId in replacements.items():
+            del data["codeInfo"][newId]
+            data["codeInfoColChildren"][existingId] = [
+                replacements.get(x, x) for x in data["codeInfoColChildren"][newId]
+            ]
+            data["codeInfoColChildren"][None] = [
+                replacements.get(x, x) for x in data["codeInfoColChildren"][None]
+            ]
+            del data["codeInfoColChildren"][newId]
+            data["sectionVariantValues"][existingId] = data["sectionVariantValues"][
+                newId
+            ]
+            del data["sectionVariantValues"][newId]
+
+        variantId = None
+        for vid in section["variantIds"]:
+            if sectionVariants[vid]["name"] == filename:
+                variantId = vid
+                break
+        if not variantId:
+            variantId, section_variant = new_section_variant(
+                name=filename,
+            )
+            section["variantIds"].append(variantId)
+            sectionVariants[variantId] = section_variant
+
+        curr_section_variant = sectionVariants[variantId]
+        curr_section_variant["values"] = data["sectionVariantValues"]
+        codeInfoColId = curr_section_variant["codeInfoColId"]
+
+        if not codeInfoColId:
+            codeInfoColId = create_new_id()
+            curr_section_variant["codeInfoColId"] = codeInfoColId
+            codeInfoCols[codeInfoColId] = new_code_info_col()
+
+        codeInfoCols[codeInfoColId]["children"] = data["codeInfoColChildren"]
+
+        section["configLanguage"] = "json"
+        filenames_loaded.add(filename)
+
+    return filenames_loaded
+
+
 def load_json_source(filename_glob, jsonSourceKeys):
     new_data = {}
 
@@ -289,83 +332,97 @@ def load_json_source(filename_glob, jsonSourceKeys):
         try:
             with open(filename, "r") as f:
                 jsonSourceData = json.load(f)
-        except:
+        except json.JSONDecodeError:
+            jsonSourceData = {}
+        except FileNotFoundError:
             logger.debug(f"couldn't find: {filename}")
             continue
 
-        new_data[filename] = {
-            "codeInfo": {},
-            "codeInfoColChildren": {},
-            "sectionVariantValues": {},
-        }
-
-        try:
-            for k in jsonSourceKeys:
-                jsonSourceData = jsonSourceData[k]
-        except:
-            # TODO print warning or something
-            pass
-
-        process_json_object(
-            new_data, jsonSourceData, filename, filename_glob, jsonSourceKeys
+        new_data[filename] = process_json_object(
+            jsonSourceData, filename, jsonSourceKeys
         )
 
-        if len(jsonSourceKeys) > 0:
-            # point directly from None (the root) to the children
-            highest_key = create_json_code_info_key(filename_glob, jsonSourceKeys)
-            codeInfoChildren = new_data[filename]["codeInfoColChildren"]
-            codeInfoChildren[None] = codeInfoChildren[highest_key]
-            del codeInfoChildren[highest_key]
+    return new_data
+
+
+def process_json_object(jsonSourceData, filename, jsonSourceKeys):
+    new_data = {
+        "codeInfo": {},
+        "codeInfoColChildren": {},
+        "sectionVariantValues": {},
+    }
+
+    try:
+        for k in jsonSourceKeys:
+            jsonSourceData = jsonSourceData[k]
+    except:
+        # TODO print warning or something
+        pass
+
+    metadataToId = {}
+
+    highest_key = process_json_object_helper(
+        new_data, jsonSourceData, filename, jsonSourceKeys, metadataToId
+    )
+
+    if len(jsonSourceKeys) > 0:
+        # point directly from None (the root) to the children
+        codeInfoChildren = new_data["codeInfoColChildren"]
+        codeInfoChildren[None] = codeInfoChildren[highest_key]
+        del codeInfoChildren[highest_key]
 
     return new_data
 
 
-def process_json_object(output, obj, filename, filename_glob, current_path):
+def process_json_object_helper(output, obj, filename, current_path, metadataToId):
     if not isinstance(obj, dict):
         return
 
     children_keys = []
     for k, v in obj.items():
         path = current_path + [k]
-        full_key, is_dict = create_json_code_info(
-            filename, filename_glob, path, k, v, output
-        )
-        children_keys.append(full_key)
+        paramInfoId, is_dict = create_json_code_info(filename, k, v, output)
+        metadataToId[createMetadataJsonString(filename, path)] = paramInfoId
+        children_keys.append(paramInfoId)
         if is_dict:
-            process_json_object(output, v, filename, filename_glob, path)
+            process_json_object_helper(output, v, filename, path, metadataToId)
 
     parent_id = None
     if len(current_path) > 0:
-        parent_id = create_json_code_info_key(filename_glob, current_path)
+        metadata = createMetadataJsonString(filename, current_path)
+        parent_id = metadataToId.get(metadata)
+        if not parent_id:
+            parent_id = create_new_id()
+            metadataToId[metadata] = parent_id
 
-    output[filename]["codeInfoColChildren"][parent_id] = children_keys
+    output["codeInfoColChildren"][parent_id] = children_keys
+    return parent_id
 
 
-def create_json_code_info(filename, filename_glob, path, key, value, output):
-    full_key = create_json_code_info_key(filename_glob, path)
+def create_json_code_info(filename, key, value, output):
+    paramInfoId = create_new_id()
     # Create code info entry
-    output[filename]["codeInfo"][full_key] = with_code_info_defaults(
-        id=full_key, name=key
+    output["codeInfo"][paramInfoId] = with_code_info_defaults(
+        id=paramInfoId,
+        name=key,
+        editable=True,
+        jsonSource=filename,
     )
-    output[filename]["codeInfoColChildren"][full_key] = []
+    output["codeInfoColChildren"][paramInfoId] = []
 
     is_dict = isinstance(value, dict)
     # Create variant value entry
     if is_dict:
         # For objects, store empty value and process children
-        output[filename]["sectionVariantValues"][full_key] = new_ev_entry()
+        output["sectionVariantValues"][paramInfoId] = new_ev_entry()
     else:
         # For non-objects, store the value directly
-        output[filename]["sectionVariantValues"][full_key] = new_ev_entry(
+        output["sectionVariantValues"][paramInfoId] = new_ev_entry(
             value=json.dumps(value)
         )
 
-    return full_key, is_dict
+    return paramInfoId, is_dict
 
 
-def create_json_code_info_key(filename_glob, path):
-    # specify separators to make json.dumps equivalent to JSON.stringify
-    key = json.dumps(
-        {"filenameGlob": filename_glob, "key": path}, separators=(",", ":")
-    )
-    return f"{C.JSON_SOURCE_PREFIX}{key}"
+def createMetadataJsonString(filename, path):
+    return json.dumps({"filename": filename, "key": path})

setta/database/settings_file.py

@@ -110,5 +110,5 @@ class MetaSettingsFile:
 
     def save_settings_project(self, p):
         save_json_source_data(p)
-        filter_data_for_json_export(p, keepCodeInfoThatHaveUITypes=False)
+        filter_data_for_json_export(p)
         save_json_to_file(self.path_meta_settings, p)