setta 0.0.9.dev3__py3-none-any.whl → 0.0.10.dev1__py3-none-any.whl

setta/__init__.py

@@ -1 +1 @@
-__version__ = "0.0.9.dev3"
+__version__ = "0.0.10.dev1"

setta/database/db/projects/load.py

@@ -128,9 +128,7 @@ def load_project_config(db, project_config_name, do_load_json_sources=True):
     load_ev_refs_into_data_structures(db, sectionVariants, codeInfo)
     load_template_vars_into_data_structures(db, sectionVariants)
     if do_load_json_sources:
-        load_json_sources_into_data_structures(
-            sections, codeInfo, codeInfoCols, sectionVariants
-        )
+        load_json_sources_into_data_structures(codeInfo, codeInfoCols, sectionVariants)
 
     return {
         "projectConfig": projectConfig,

setta/database/db/projects/save.py

@@ -15,7 +15,7 @@ from setta.utils.generate_memorable_string import generate_memorable_available_s
 from setta.utils.utils import filter_dict
 
 from ..codeInfo.save import save_code_info, save_code_info_col
-from ..sections.jsonSource import remove_json_source_data, save_json_source_data
+from ..sections.jsonSource import remove_json_source_values, save_json_source_data
 from ..sections.save import save_sections
 from ..sectionVariants.save import save_section_variants
 from ..uiTypes.save import save_ui_type_cols, save_ui_types
@@ -24,7 +24,7 @@ from ..uiTypes.save import save_ui_type_cols, save_ui_types
 def save_project_details(db, p, do_save_json_source_data=True):
     if do_save_json_source_data:
         save_json_source_data(p)
-    remove_json_source_data(p)
+    remove_json_source_values(p)
     save_artifacts(db, p["artifacts"])
     save_artifact_groups(db, p["artifactGroups"], p["sections"])
     save_code_info(db, p["codeInfo"])

setta/database/db/projects/utils.py

@@ -5,7 +5,7 @@ from setta.database.db.codeInfo.utils import (
     add_defaults_to_code_info,
     add_defaults_to_code_info_cols,
 )
-from setta.database.db.sections.jsonSource import remove_json_source_data
+from setta.database.db.sections.jsonSource import remove_json_source_values
 from setta.database.db.sections.load import load_json_sources_into_data_structures
 from setta.database.db.sections.utils import add_defaults_to_sections
 from setta.database.db.sectionVariants.utils import add_defaults_to_section_variants
@@ -35,7 +35,7 @@ def add_defaults_to_project(p):
 def add_defaults_to_project_and_load_json_sources(p):
     add_defaults_to_project(p)
     load_json_sources_into_data_structures(
-        p["sections"], p["codeInfo"], p["codeInfoCols"], p["sectionVariants"]
+        p["codeInfo"], p["codeInfoCols"], p["sectionVariants"]
     )
 
 
@@ -44,7 +44,7 @@ def remove_empty(x):
 
 
 def filter_data_for_json_export(p):
-    remove_json_source_data(p)
+    remove_json_source_values(p)
 
     p["projectConfig"] = filter_dict(
         p["projectConfig"],

setta/database/db/sectionVariants/copy.py

@@ -2,19 +2,11 @@ from setta.database.utils import remap_ids, rename_keys
 from setta.utils.generate_memorable_string import generate_memorable_string
 
 
-def copy_section_variants(
-    sections, section_variants, code_info_id_map, code_info_col_id_map
-):
+def copy_section_variants(section_variants, code_info_id_map, code_info_col_id_map):
     new_section_variants, section_variant_id_map = remap_ids(section_variants)
-    keep_old_name = set()
-    for s in sections.values():
-        if not s["jsonSource"]:
-            continue
-        for v in s["variantIds"]:
-            keep_old_name.add(section_variant_id_map[v])
 
-    for id, obj in new_section_variants.items():
-        if id not in keep_old_name:
+    for obj in new_section_variants.values():
+        if not obj["isJsonSource"]:
             obj["name"] = generate_memorable_string()
         obj["values"] = rename_keys(obj["values"], code_info_id_map)
         if obj["codeInfoColId"]:

setta/database/db/sections/copy.py

@@ -44,7 +44,7 @@ def copy_sections_and_other_info(x):
         x["codeInfoCols"], code_info_id_map
     )
     new_section_variants, section_variant_id_map = copy_section_variants(
-        x["sections"], x["sectionVariants"], code_info_id_map, code_info_col_id_map
+        x["sectionVariants"], code_info_id_map, code_info_col_id_map
     )
     new_ui_types, ui_type_id_map = copy_ui_types(x["uiTypes"])
     new_ui_type_cols, ui_type_col_id_map = copy_ui_type_cols(

setta/database/db/sections/jsonSource.py

@@ -1,77 +1,61 @@
-import json
 from collections import defaultdict
 
-from setta.utils.constants import BASE_UI_TYPE_IDS, C
-from setta.utils.utils import (
-    recursive_dict_merge,
-    replace_null_keys_with_none,
-    save_json_to_file,
-    try_json,
-)
+from setta.utils.utils import replace_null_keys_with_none, save_json_to_file, try_json
 
 
-def save_json_source_data(p, section_ids=None, forking_from=None):
-    sections = {
+def save_json_source_data(p, variant_ids=None):
+    variants = {
         k: v
-        for k, v in p["sections"].items()
-        if v.get("jsonSource", None) and ((not section_ids) or k in section_ids)
+        for k, v in p["sectionVariants"].items()
+        if ((not variant_ids) or k in variant_ids)
     }
 
     to_be_saved = defaultdict(dict)
 
-    forking_from_data = {}
-    if forking_from:
-        with open(forking_from, "r") as f:
-            forking_from_data = json.load(f)
-
     p["codeInfoCols"] = replace_null_keys_with_none(p["codeInfoCols"])
 
-    ancestor_paths = build_ancestor_paths(p["codeInfo"], p["codeInfoCols"])
-
-    for s in sections.values():
-        if not s["jsonSource"] or s["jsonSourceMissing"]:
+    for variant in variants.values():
+        if not variant["isJsonSource"]:
             continue
+        codeInfoColId = variant["codeInfoColId"]
+        if not codeInfoColId:
+            continue
+        codeInfoCol = p["codeInfoCols"][codeInfoColId]
+        filename = variant["name"]
+
+        ancestor_paths = build_ancestor_paths(p["codeInfo"], codeInfoCol["children"])
+        recursively_add_keys(
+            p,
+            variant,
+            codeInfoCol,
+            to_be_saved[filename],
+            None,
+            variant["jsonSourceKeys"],
+            ancestor_paths,
+        )
 
-        for variantId in s["variantIds"]:
-            variant = p["sectionVariants"][variantId]
-            codeInfoColId = variant["codeInfoColId"]
-            codeInfoCol = p["codeInfoCols"][codeInfoColId]
-            filename = variant["name"]
-
-            recursively_add_keys(
-                p,
-                variant,
-                codeInfoCol,
-                to_be_saved[filename],
-                None,
-                s["jsonSourceKeys"],
-                ancestor_paths,
-            )
-
-            # Make sure the jsonSourceKeys are present.
-            # (They might not be because they are completely empty)
-            add_key_path_to_dict(to_be_saved[filename], s["jsonSourceKeys"])
+        # Make sure the jsonSourceKeys are present.
+        # (They might not be because they are completely empty)
+        add_key_path_to_dict(to_be_saved[filename], variant["jsonSourceKeys"])
 
     # Save each file
     for filename, data in to_be_saved.items():
-        data = recursive_dict_merge(forking_from_data, data)
         save_json_to_file(filename, data)
 
     return to_be_saved
 
 
-def build_ancestor_paths(codeInfo, codeInfoCols):
+def build_ancestor_paths(codeInfo, codeInfoColChildren):
     parent_map = {}
-    for col in codeInfoCols.values():
-        for parent_id, children in col["children"].items():
-            for child_id in children:
-                parent_map[(codeInfo[child_id]["jsonSource"], child_id)] = parent_id
+    for parent_id, children in codeInfoColChildren.items():
+        for child_id in children:
+            parent_map[child_id] = parent_id
 
     ancestor_paths = {}
-    for node_id in codeInfo:
-        if node_id not in ancestor_paths:
+    for id in parent_map.keys():
+        if id not in ancestor_paths:
             path = []
-            current_id = node_id
+            current_id = id
 
             # Traverse up to build the path
             while current_id is not None:
@@ -80,14 +64,12 @@ def build_ancestor_paths(codeInfo, codeInfoCols):
                     path.insert(0, name)
 
                     # Get parent using the map
-                    parent_id = parent_map.get(
-                        (codeInfo[current_id]["jsonSource"], current_id)
-                    )
+                    parent_id = parent_map.get(current_id)
                     current_id = parent_id
                 else:
                     break
 
-            ancestor_paths[node_id] = path
+            ancestor_paths[id] = path
 
     return ancestor_paths
 
@@ -97,20 +79,16 @@ def recursively_add_keys(
 ):
     for k in codeInfoCol["children"][codeInfoId]:
         children = codeInfoCol["children"][k]
-        json_source = p["codeInfo"][k].get("jsonSource")
 
-        if json_source:
-            # Get pre-computed key path
-            key_path = [*jsonSourceKeys, *ancestor_paths[k]]
-            value = try_getting_value(variant, k, children)
+        # Get pre-computed key path
+        key_path = [*jsonSourceKeys, *ancestor_paths[k]]
+        value = try_getting_value(variant, k, children)
+        current_dict = add_key_path_to_dict(input_dict, key_path[:-1])
 
-            current_dict = add_key_path_to_dict(input_dict, key_path[:-1])
+        # Set the value at the final position
+        if key_path:  # Only set if we have a path
+            current_dict[key_path[-1]] = value
 
-            # Set the value at the final position
-            if key_path:  # Only set if we have a path
-                current_dict[key_path[-1]] = value
-
-        # Continue recursion regardless of whether this node has a jsonSource
         recursively_add_keys(
             p, variant, codeInfoCol, input_dict, k, jsonSourceKeys, ancestor_paths
         )
@@ -133,54 +111,7 @@ def try_getting_value(variant, codeInfoId, codeInfoChildren):
     return {}
 
 
-def condition_keep_code_info(codeInfo, jsonCodeInfoWithUIType):
-    if not codeInfo:
-        return False
-    return not codeInfo["jsonSource"] or codeInfo["id"] in jsonCodeInfoWithUIType
-
-
-def remove_json_source_data(p):
+def remove_json_source_values(p):
     for variant in p["sectionVariants"].values():
-        variant["values"] = {
-            k: v
-            for k, v in variant["values"].items()
-            if not p["codeInfo"][k]["jsonSource"]
-        }
-
-    jsonCodeInfoWithUIType = set()
-    for uiTypeCol in p["uiTypeCols"].values():
-        for paramInfoId, uiTypeInfo in uiTypeCol.items():
-            # we want to know which json source params have an associated uiTypeId
-            # only if it's not the base TEXT type, since that's the default
-            if (
-                p["codeInfo"][paramInfoId]["jsonSource"]
-                and uiTypeInfo["uiTypeId"] != BASE_UI_TYPE_IDS[C.TEXT]
-            ):
-                jsonCodeInfoWithUIType.add(paramInfoId)
-
-    p["codeInfo"] = {
-        k: v
-        for k, v in p["codeInfo"].items()
-        if condition_keep_code_info(v, jsonCodeInfoWithUIType)
-    }
-
-    for codeInfoColId in p["codeInfoCols"].keys():
-        codeInfoCol = p["codeInfoCols"][codeInfoColId]
-        codeInfoCol["children"] = {
-            k: v
-            for k, v in codeInfoCol["children"].items()
-            if k is None
-            or condition_keep_code_info(
-                p["codeInfo"].get(k),
-                jsonCodeInfoWithUIType,
-            )
-        }
-        for id, children in codeInfoCol["children"].items():
-            codeInfoCol["children"][id] = [
-                c
-                for c in children
-                if condition_keep_code_info(
-                    p["codeInfo"].get(c),
-                    jsonCodeInfoWithUIType,
-                )
-            ]
+        if variant["isJsonSource"]:
+            variant["values"] = {}

setta/database/db/sections/load.py

@@ -1,5 +1,4 @@
 import copy
-import glob
 import itertools
 import json
 import logging
@@ -9,7 +8,7 @@ from setta.database.db.artifacts.load import load_artifact_groups
 from setta.database.db.codeInfo.utils import new_code_info_col, with_code_info_defaults
 from setta.database.db.sections.jsonSource import build_ancestor_paths
 from setta.database.db.sections.utils import with_section_defaults
-from setta.database.db.sectionVariants.utils import new_ev_entry, new_section_variant
+from setta.database.db.sectionVariants.utils import new_ev_entry
 from setta.database.utils import create_new_id
 
 from ..sectionVariants.load import load_section_variants
@@ -201,148 +200,84 @@ def num_projects_each_section_exists_in(db, section_ids):
 
 
 def load_json_sources_into_data_structures(
-    sections, codeInfo, codeInfoCols, sectionVariants, section_ids=None
+    codeInfo, codeInfoCols, sectionVariants, variant_ids=None
 ):
-    filenames_loaded = set()
-    sections = {
+    sectionVariants = {
         k: v
-        for k, v in sections.items()
-        if v["jsonSource"] and ((not section_ids) or k in section_ids)
+        for k, v in sectionVariants.items()
+        if ((not variant_ids) or k in variant_ids)
     }
-    for s in sections.values():
-        logger.debug(
-            f'Attempting to read {s["jsonSource"]} with keys {s["jsonSourceKeys"]}'
-        )
-        new_data = load_json_source(s["jsonSource"], s["jsonSourceKeys"])
-        filenames_loaded.update(
-            merge_into_existing(new_data, s, sectionVariants, codeInfo, codeInfoCols)
-        )
 
-    # delete variants that aren't associated with a loaded file
-    to_delete = []
-    for s in sections.values():
-        for vid in s["variantIds"]:
-            if sectionVariants[vid]["name"] not in filenames_loaded:
-                logger.debug(
-                    f'Removing variant {sectionVariants[vid]["name"]} because the associated json was not found'
-                )
-                to_delete.append(vid)
+    for variant in sectionVariants.values():
+        if not variant["isJsonSource"]:
+            continue
+        new_data, isMissing = load_json_source(
+            variant["name"], variant["jsonSourceKeys"]
+        )
+        variant["jsonSourcMissing"] = isMissing
+        if not isMissing:
+            if not variant["codeInfoColId"]:
+                variant["codeInfoColId"] = create_new_id()
+                codeInfoCols[variant["codeInfoColId"]] = new_code_info_col()
+            codeInfoCol = codeInfoCols[variant["codeInfoColId"]]
+            merge_into_existing(new_data, variant, codeInfo, codeInfoCol)
 
-    for vid in to_delete:
-        if sectionVariants[vid]["codeInfoColId"]:
-            del codeInfoCols[sectionVariants[vid]["codeInfoColId"]]
-        del sectionVariants[vid]
 
-    for s in sections.values():
-        s["jsonSourceMissing"] = False
-        s["variantIds"] = [v for v in s["variantIds"] if v in sectionVariants]
-        if len(s["variantIds"]) == 0:
-            logger.debug("Section has no variantIds. Creating new section variant.")
-            variantId, variant = new_section_variant()
-            sectionVariants[variantId] = variant
-            s["variantId"] = variantId
-            s["variantIds"].append(variantId)
-            s["jsonSourceMissing"] = True
-        elif s["variantId"] not in s["variantIds"]:
-            logger.debug(
-                "Selected variantId is not in list of variantIds. Changing selected variantId"
-            )
-            s["variantId"] = s["variantIds"][0]
-
-        if s["defaultVariantId"] not in s["variantIds"]:
-            logger.debug(
-                "Default variantId is not in list of variantIds. Changing default variantId"
-            )
-            s["defaultVariantId"] = s["variantId"]
-
-
-def merge_into_existing(new_data, section, sectionVariants, codeInfo, codeInfoCols):
-    filenames_loaded = set()
+def merge_into_existing(data, sectionVariant, codeInfo, codeInfoCol):
+    filename = sectionVariant["name"]
     jsonSourceMetadata_to_id = {}
-    ancestor_paths = build_ancestor_paths(codeInfo, codeInfoCols)
-    for id, info in codeInfo.items():
+    ancestor_paths = build_ancestor_paths(codeInfo, codeInfoCol["children"])
+    for id in ancestor_paths:
         jsonSourceMetadata_to_id[
-            createMetadataJsonString(info["jsonSource"], ancestor_paths[id])
+            createMetadataJsonString(filename, ancestor_paths[id])
         ] = id
 
-    for filename, data in new_data.items():
-        replacements = {}
-        new_ancestor_paths = build_ancestor_paths(
-            data["codeInfo"], {None: {"children": data["codeInfoColChildren"]}}
+    replacements = {}
+    new_ancestor_paths = build_ancestor_paths(
+        data["codeInfo"], data["codeInfoColChildren"]
+    )
+    for newId, newInfo in data["codeInfo"].items():
+        existingId = jsonSourceMetadata_to_id.get(
+            createMetadataJsonString(filename, new_ancestor_paths[newId])
         )
-        for newId, newInfo in data["codeInfo"].items():
-            existingId = jsonSourceMetadata_to_id.get(
-                createMetadataJsonString(
-                    newInfo["jsonSource"], new_ancestor_paths[newId]
-                )
-            )
-            if existingId:
-                replacements[newId] = existingId
-            else:
-                codeInfo[newId] = newInfo
-
-        for newId, existingId in replacements.items():
-            del data["codeInfo"][newId]
-            data["codeInfoColChildren"][existingId] = [
-                replacements.get(x, x) for x in data["codeInfoColChildren"][newId]
-            ]
-            data["codeInfoColChildren"][None] = [
-                replacements.get(x, x) for x in data["codeInfoColChildren"][None]
-            ]
-            del data["codeInfoColChildren"][newId]
-            data["sectionVariantValues"][existingId] = data["sectionVariantValues"][
-                newId
-            ]
-            del data["sectionVariantValues"][newId]
-
-        variantId = None
-        for vid in section["variantIds"]:
-            if sectionVariants[vid]["name"] == filename:
-                variantId = vid
-                break
-        if not variantId:
-            variantId, section_variant = new_section_variant(
-                name=filename,
-            )
-            section["variantIds"].append(variantId)
-            sectionVariants[variantId] = section_variant
-
-        curr_section_variant = sectionVariants[variantId]
-        curr_section_variant["values"] = data["sectionVariantValues"]
-        codeInfoColId = curr_section_variant["codeInfoColId"]
-
-        if not codeInfoColId:
-            codeInfoColId = create_new_id()
-            curr_section_variant["codeInfoColId"] = codeInfoColId
-            codeInfoCols[codeInfoColId] = new_code_info_col()
-
-        codeInfoCols[codeInfoColId]["children"] = data["codeInfoColChildren"]
-
-        section["configLanguage"] = "json"
-        filenames_loaded.add(filename)
-
-    return filenames_loaded
-
-
-def load_json_source(filename_glob, jsonSourceKeys):
-    new_data = {}
-
-    filenames = glob.glob(filename_glob)
-    for filename in filenames:
-        try:
-            with open(filename, "r") as f:
-                jsonSourceData = json.load(f)
-        except json.JSONDecodeError:
-            jsonSourceData = {}
-        except FileNotFoundError:
-            logger.debug(f"couldn't find: {filename}")
-            continue
+        if existingId:
+            replacements[newId] = existingId
+        else:
+            codeInfo[newId] = newInfo
+
+    for newId, existingId in replacements.items():
+        del data["codeInfo"][newId]
+        data["codeInfoColChildren"][existingId] = [
+            replacements.get(x, x) for x in data["codeInfoColChildren"][newId]
+        ]
+        data["codeInfoColChildren"][None] = [
+            replacements.get(x, x) for x in data["codeInfoColChildren"][None]
+        ]
+        del data["codeInfoColChildren"][newId]
+        data["sectionVariantValues"][existingId] = data["sectionVariantValues"][newId]
+        del data["sectionVariantValues"][newId]
+
+    sectionVariant["values"] = data["sectionVariantValues"]
+    codeInfoCol["children"] = data["codeInfoColChildren"]
+
+
+def load_json_source(filename, jsonSourceKeys):
+    jsonSourceData = {}
+    isMissing = False
 
-        new_data[filename] = process_json_object(
-            jsonSourceData, filename, jsonSourceKeys
-        )
+    try:
+        logger.debug(f"Attempting to read {filename} with keys {jsonSourceKeys}")
+        with open(filename, "r") as f:
+            jsonSourceData = json.load(f)
+    except json.JSONDecodeError:
+        pass
+    except FileNotFoundError:
+        logger.debug(f"couldn't find: {filename}")
+        isMissing = True
 
-    return new_data
+    new_data = process_json_object(jsonSourceData, filename, jsonSourceKeys)
+
+    return new_data, isMissing
 
 
 def process_json_object(jsonSourceData, filename, jsonSourceKeys):
@@ -381,7 +316,7 @@ def process_json_object_helper(output, obj, filename, current_path, metadataToId
     children_keys = []
     for k, v in obj.items():
         path = current_path + [k]
-        paramInfoId, is_dict = create_json_code_info(filename, k, v, output)
+        paramInfoId, is_dict = create_json_code_info(k, v, output)
         metadataToId[createMetadataJsonString(filename, path)] = paramInfoId
         children_keys.append(paramInfoId)
         if is_dict:
@@ -399,14 +334,13 @@ def process_json_object_helper(output, obj, filename, current_path, metadataToId
     return parent_id
 
 
-def create_json_code_info(filename, key, value, output):
+def create_json_code_info(key, value, output):
     paramInfoId = create_new_id()
     # Create code info entry
     output["codeInfo"][paramInfoId] = with_code_info_defaults(
         id=paramInfoId,
         name=key,
         editable=True,
-        jsonSource=filename,
     )
     output["codeInfoColChildren"][paramInfoId] = []
 

setta/database/settings_file.py

@@ -77,9 +77,9 @@ class MetaSettingsFile:
         return settings_slice_to_path
 
     def find_path_for_settings_slice(self, p, slice):
-        for sectionId, section in p["sections"].items():
-            if section["jsonSourceKeys"] == [slice]:
-                variant = get_selected_section_variant(p, sectionId)
+        for sectionId in p["sections"].keys():
+            variant = get_selected_section_variant(p, sectionId)
+            if variant["jsonSourceKeys"] == [slice]:
                 return variant["name"]
         return None
 

setta/routers/sections.py

@@ -54,17 +54,16 @@ class GlobalParamSweepSectionToYamlRequest(BaseModel):
 
 class LoadSectionJSONSourceRequest(BaseModel):
     project: dict
-    sectionIdToJSONSource: Dict[str, str]
+    variantIdsToLoad: List[str]
 
 
 class SaveSectionJSONSourceRequest(BaseModel):
     project: dict
-    sectionId: str
-    forking_from: str
+    variantId: str
 
 
 class NewJSONVersionNameRequest(BaseModel):
-    filenameGlob: str
+    filename: str
 
 
 class CreateFileRequest(BaseModel):
@@ -134,21 +133,18 @@ def route_global_param_sweep_section_to_yaml(x: GlobalParamSweepSectionToYamlReq
 @router.post(C.ROUTE_LOAD_SECTION_JSON_SOURCE)
 def route_load_section_json_source(x: LoadSectionJSONSourceRequest):
     p = x.project
-    for k, v in x.sectionIdToJSONSource.items():
-        p["sections"][k]["jsonSource"] = v
     load_json_sources_into_data_structures(
-        p["sections"],
         p["codeInfo"],
         p["codeInfoCols"],
         p["sectionVariants"],
-        section_ids=list(x.sectionIdToJSONSource.keys()),
+        variant_ids=x.variantIdsToLoad,
     )
     return {"project": p}
 
 
 @router.post(C.ROUTE_NEW_JSON_VERSION_NAME)
 def route_new_json_version_name(x: NewJSONVersionNameRequest):
-    new_filename = generate_new_filename(x.filenameGlob)
+    new_filename = generate_new_filename(x.filename)
     Path(new_filename).parent.mkdir(parents=True, exist_ok=True)
     Path(new_filename).touch()
     return new_filename
@@ -162,7 +158,7 @@ def route_create_file(x: CreateFileRequest):
 
 @router.post(C.ROUTE_SAVE_SECTION_JSON_SOURCE)
 def route_save_section_json_source(x: SaveSectionJSONSourceRequest):
-    save_json_source_data(x.project, [x.sectionId], x.forking_from)
+    save_json_source_data(x.project, [x.variantId])
 
 
 @router.post(C.ROUTE_GET_JSON_SOURCE_PATH_TO_BE_DELETED)