jsonjsdb 0.7.3__tar.gz → 0.8.0__tar.gz

{jsonjsdb-0.7.3 → jsonjsdb-0.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: jsonjsdb
-Version: 0.7.3
+Version: 0.8.0
 Summary: Python library for JSONJS database loading
 Project-URL: Homepage, https://github.com/datannur/jsonjsdb
 Project-URL: Repository, https://github.com/datannur/jsonjsdb
@@ -172,6 +172,9 @@ db.save()
 # Disable tracking
 db.save(track_evolution=False)
 
+# Skip .json.js files (faster, smaller output)
+db.save(write_js=False)
+
 # Use Excel as source (for easy editing of logs)
 db.save(evolution_xlsx=Path("path/to/evolution.xlsx"))
 
@@ -179,6 +182,27 @@ db.save(evolution_xlsx=Path("path/to/evolution.xlsx"))
 db.save(timestamp=1741186800)
 ```
 
+#### Cascade Filtering
+
+When a parent entity is added or deleted, all child entities are also added/deleted.
+By default, this creates noise in the evolution log. Use `parent_relations` to automatically
+filter out cascade entries:
+
+```python
+db.save(
+    parent_relations={
+        "variable": "dataset",    # variable.dataset_id → dataset
+        "freq": "variable",       # freq.variable_id → variable
+    }
+)
+```
+
+With cascade filtering:
+- Adding a dataset with 50 variables logs only 1 entry (the dataset add)
+- Deleting a dataset logs only the parent delete, not all child deletes
+- Updates are always logged (no filtering)
+- Explicit child additions (to existing parent) are still logged
+
 When `evolution_xlsx` is provided:
 - The xlsx file becomes the source of truth (read from xlsx if it exists)
 - User edits made in Excel are preserved on subsequent saves
@@ -193,6 +217,7 @@ Evolution format:
     "entity": "user",
     "entity_id": "user_2",
     "parent_entity_id": null,
+    "parent_entity": null,
     "variable": null,
     "old_value": null,
     "new_value": null,
@@ -201,12 +226,13 @@ Evolution format:
   {
     "timestamp": 1741186800,
     "type": "update",
-    "entity": "user",
-    "entity_id": "user_1",
-    "parent_entity_id": null,
-    "variable": "score",
-    "old_value": 100,
-    "new_value": 200,
+    "entity": "variable",
+    "entity_id": "var_1",
+    "parent_entity_id": "ds_1",
+    "parent_entity": "dataset",
+    "variable": "name",
+    "old_value": "Old Name",
+    "new_value": "New Name",
     "name": null
   }
 ]

{jsonjsdb-0.7.3 → jsonjsdb-0.8.0}/README.md

@@ -147,6 +147,9 @@ db.save()
 # Disable tracking
 db.save(track_evolution=False)
 
+# Skip .json.js files (faster, smaller output)
+db.save(write_js=False)
+
 # Use Excel as source (for easy editing of logs)
 db.save(evolution_xlsx=Path("path/to/evolution.xlsx"))
 
@@ -154,6 +157,27 @@ db.save(evolution_xlsx=Path("path/to/evolution.xlsx"))
 db.save(timestamp=1741186800)
 ```
 
+#### Cascade Filtering
+
+When a parent entity is added or deleted, all child entities are also added/deleted.
+By default, this creates noise in the evolution log. Use `parent_relations` to automatically
+filter out cascade entries:
+
+```python
+db.save(
+    parent_relations={
+        "variable": "dataset",    # variable.dataset_id → dataset
+        "freq": "variable",       # freq.variable_id → variable
+    }
+)
+```
+
+With cascade filtering:
+- Adding a dataset with 50 variables logs only 1 entry (the dataset add)
+- Deleting a dataset logs only the parent delete, not all child deletes
+- Updates are always logged (no filtering)
+- Explicit child additions (to existing parent) are still logged
+
 When `evolution_xlsx` is provided:
 - The xlsx file becomes the source of truth (read from xlsx if it exists)
 - User edits made in Excel are preserved on subsequent saves
@@ -168,6 +192,7 @@ Evolution format:
     "entity": "user",
     "entity_id": "user_2",
     "parent_entity_id": null,
+    "parent_entity": null,
     "variable": null,
     "old_value": null,
     "new_value": null,
@@ -176,12 +201,13 @@ Evolution format:
   {
     "timestamp": 1741186800,
     "type": "update",
-    "entity": "user",
-    "entity_id": "user_1",
-    "parent_entity_id": null,
-    "variable": "score",
-    "old_value": 100,
-    "new_value": 200,
+    "entity": "variable",
+    "entity_id": "var_1",
+    "parent_entity_id": "ds_1",
+    "parent_entity": "dataset",
+    "variable": "name",
+    "old_value": "Old Name",
+    "new_value": "New Name",
     "name": null
   }
 ]

{jsonjsdb-0.7.3 → jsonjsdb-0.8.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "jsonjsdb"
-version = "0.7.3"
+version = "0.8.0"
 description = "Python library for JSONJS database loading"
 authors = [{ name = "datannur" }]
 readme = "README.md"

{jsonjsdb-0.7.3 → jsonjsdb-0.8.0}/src/jsonjsdb/database.py

@@ -10,6 +10,7 @@ import polars as pl
 from .evolution import (
     EvolutionEntry,
     compare_datasets,
+    filter_cascade_entries,
     get_timestamp,
     load_evolution,
     save_evolution,
@@ -107,6 +108,8 @@ class Jsonjsdb:
         track_evolution: bool = True,
         evolution_xlsx: Path | str | None = None,
         timestamp: int | None = None,
+        write_js: bool = True,
+        parent_relations: dict[str, str] | None = None,
     ) -> None:
         """Save all tables to disk with optional evolution tracking.
 
@@ -118,6 +121,9 @@ class Jsonjsdb:
             track_evolution: Enable change tracking (default: True)
             evolution_xlsx: Optional path for evolution.xlsx output
             timestamp: Optional timestamp override for deterministic outputs
+            write_js: If True, write both .json and .json.js (default: True)
+            parent_relations: Child->parent table mapping for cascade filtering
+                Example: {"variable": "dataset", "freq": "variable"}
         """
         save_path = Path(path) if path else self._path
 
@@ -145,26 +151,35 @@ class Jsonjsdb:
             # Track evolution if enabled
             if track_evolution:
                 old_df = self._get_old_table(save_path, name, same_path)
-                entries = compare_datasets(old_df, persistable_df, ts, name)
+                entries = compare_datasets(
+                    old_df, persistable_df, ts, name, parent_relations
+                )
                 new_entries.extend(entries)
 
             write_table_json(persistable_df, save_path / f"{name}.json")
-            write_table_jsonjs(persistable_df, name, save_path / f"{name}.json.js")
+            if write_js:
+                write_table_jsonjs(persistable_df, name, save_path / f"{name}.json.js")
             table_names.append(name)
 
             # Update snapshot for next comparison
             self._original_snapshots[name] = persistable_df.clone()
 
         # Save evolution if there are new entries
+        if track_evolution and new_entries:
+            # Filter cascade entries (child add/delete when parent has same operation)
+            new_entries = filter_cascade_entries(new_entries)
+
         if track_evolution and new_entries:
             xlsx_path = Path(evolution_xlsx) if evolution_xlsx else None
             existing_entries = load_evolution(save_path, xlsx_path)
             all_entries = existing_entries + new_entries
             save_evolution(all_entries, save_path, xlsx_path)
-            if "evolution" not in table_names:
+            if "evolution" not in table_names:  # pragma: no branch
                 table_names.append("evolution")
 
-        write_table_index(table_names, save_path / "__table__.json", ts)
+        write_table_index(
+            table_names, save_path / "__table__.json", ts, write_js=write_js
+        )
 
         self._path = save_path
 

{jsonjsdb-0.7.3 → jsonjsdb-0.8.0}/src/jsonjsdb/evolution.py

@@ -27,6 +27,7 @@ class EvolutionEntry:
     entity: str
     entity_id: str | int
     parent_entity_id: str | int | None
+    parent_entity: str | None
     variable: str | None
     old_value: Any
     new_value: Any
@@ -45,13 +46,40 @@ def _standardize_id(id_value: str) -> str:
     return INVALID_ID_PATTERN.sub("", trimmed)
 
 
-def _get_first_parent_id(row: dict[str, Any]) -> str | int | None:
-    """Find the first foreign key column (_id or Id suffix) and return its value."""
+def _get_parent_info(
+    row: dict[str, Any],
+    entity: str,
+    parent_relations: dict[str, str] | None,
+) -> tuple[str | None, str | int | None]:
+    """Get parent entity and id based on config or FK convention.
+
+    Args:
+        row: Row data
+        entity: Current entity/table name
+        parent_relations: Mapping of child_table -> parent_table
+
+    Returns:
+        Tuple of (parent_entity, parent_entity_id)
+    """
+    if parent_relations and entity in parent_relations:
+        parent_entity = parent_relations[entity]
+        fk_col = f"{parent_entity}_id"
+        parent_id = row.get(fk_col)
+        if isinstance(parent_id, (str, int)):
+            return (parent_entity, parent_id)
+        return (parent_entity, None)
+
+    # Fallback: auto-detect from first FK column
     for key, value in row.items():
-        if key.endswith("_id") or key.endswith("Id"):
+        if key.endswith("_id") and key != "id":
+            parent_entity = key[:-3]  # strip "_id"
             if isinstance(value, (str, int)):
-                return value
-    return None
+                return (parent_entity, value)
+        elif key.endswith("Id"):
+            parent_entity = key[:-2]  # strip "Id"
+            if isinstance(value, (str, int)):
+                return (parent_entity, value)
+    return (None, None)
 
 
 def _add_composite_id_if_missing(df: pl.DataFrame) -> tuple[pl.DataFrame, bool]:
@@ -109,6 +137,7 @@ def compare_datasets(
     new_df: pl.DataFrame,
     timestamp: int,
     entity: str,
+    parent_relations: dict[str, str] | None = None,
 ) -> list[EvolutionEntry]:
     """Compare two datasets and return list of evolution entries.
 
@@ -117,6 +146,7 @@ def compare_datasets(
         new_df: New version of the data
         timestamp: Unix timestamp in seconds
         entity: Table/entity name
+        parent_relations: Mapping of child_table -> parent_table for cascade filtering
 
     Returns:
         List of EvolutionEntry objects describing the changes
@@ -130,8 +160,6 @@ def compare_datasets(
     if old_df.is_empty():
         return entries
 
-    # Normalize IDs for consistent comparison
-
     # Normalize id columns to string for consistent comparison
     old_df = _normalize_id_column(old_df)
     new_df = _normalize_id_column(new_df)
@@ -144,9 +172,7 @@ def compare_datasets(
     map_new = _df_to_dict_by_id(new_df)
 
     # Determine all variables to compare
-    if old_df.is_empty():
-        variables = new_df.columns
-    elif new_df.is_empty():
+    if new_df.is_empty():
         variables = old_df.columns
     else:
         variables = list(set(old_df.columns) | set(new_df.columns))
@@ -176,6 +202,9 @@ def compare_datasets(
             if _values_are_empty(old_value, new_value):
                 continue
 
+            parent_entity, parent_id = _get_parent_info(
+                obj_new, entity, parent_relations
+            )
             entries.append(
                 EvolutionEntry(
                     timestamp=timestamp,
@@ -187,8 +216,11 @@ def compare_datasets(
                         else entity_id
                     ),
                     parent_entity_id=(
-                        str(entity_id).split("---")[0] if has_composite_id else None
+                        str(entity_id).split("---")[0]
+                        if has_composite_id
+                        else parent_id
                     ),
+                    parent_entity=None if has_composite_id else parent_entity,
                     variable=variable,
                     old_value=old_value,
                     new_value=new_value,
@@ -199,6 +231,7 @@ def compare_datasets(
     # Detect additions
     for entity_id in ids_added:
         obj_new = map_new[entity_id]
+        parent_entity, parent_id = _get_parent_info(obj_new, entity, parent_relations)
         entries.append(
             EvolutionEntry(
                 timestamp=timestamp,
@@ -208,8 +241,9 @@ def compare_datasets(
                     _standardize_id(str(entity_id)) if has_composite_id else entity_id
                 ),
                 parent_entity_id=(
-                    str(entity_id).split("---")[0] if has_composite_id else None
+                    str(entity_id).split("---")[0] if has_composite_id else parent_id
                 ),
+                parent_entity=None if has_composite_id else parent_entity,
                 variable=None,
                 old_value=None,
                 new_value=None,
@@ -220,6 +254,7 @@ def compare_datasets(
     # Detect deletions
     for entity_id in ids_removed:
         obj_old = map_old[entity_id]
+        parent_entity, parent_id = _get_parent_info(obj_old, entity, parent_relations)
         entries.append(
             EvolutionEntry(
                 timestamp=timestamp,
@@ -228,7 +263,10 @@ def compare_datasets(
                 entity_id=(
                     _standardize_id(str(entity_id)) if has_composite_id else entity_id
                 ),
-                parent_entity_id=_get_first_parent_id(obj_old),
+                parent_entity_id=(
+                    str(entity_id).split("---")[0] if has_composite_id else parent_id
+                ),
+                parent_entity=None if has_composite_id else parent_entity,
                 variable=None,
                 old_value=None,
                 new_value=None,
@@ -243,6 +281,51 @@ def compare_datasets(
     return entries
 
 
+def filter_cascade_entries(entries: list[EvolutionEntry]) -> list[EvolutionEntry]:
+    """Filter out cascade add/delete entries where parent has same operation.
+
+    When a parent entity is added or deleted, child entities are also added/deleted.
+    This function removes child entries that are part of a cascade operation,
+    keeping only the meaningful parent-level changes.
+
+    Args:
+        entries: List of evolution entries to filter
+
+    Returns:
+        Filtered list with cascade entries removed
+    """
+    # Index parent operations: (timestamp, type, entity, entity_id)
+    parent_ops: set[tuple[int, str, str, str]] = {
+        (e.timestamp, e.type, e.entity, str(e.entity_id))
+        for e in entries
+        if e.type in ("add", "delete")
+    }
+
+    result: list[EvolutionEntry] = []
+    for entry in entries:
+        # Always keep updates
+        if entry.type == "update":
+            result.append(entry)
+            continue
+
+        # Keep entries without parent relation
+        if not entry.parent_entity or entry.parent_entity_id is None:
+            result.append(entry)
+            continue
+
+        # Check if parent has the same operation in this batch
+        parent_key = (
+            entry.timestamp,
+            entry.type,
+            entry.parent_entity,
+            str(entry.parent_entity_id),
+        )
+        if parent_key not in parent_ops:
+            result.append(entry)
+
+    return result
+
+
 def load_evolution(path: Path, xlsx_path: Path | None = None) -> list[EvolutionEntry]:
     """Load existing evolution entries.
 
@@ -266,6 +349,7 @@ def load_evolution(path: Path, xlsx_path: Path | None = None) -> list[EvolutionE
             entity=row["entity"],
             entity_id=row["entity_id"],
             parent_entity_id=row.get("parent_entity_id"),
+            parent_entity=row.get("parent_entity"),
             variable=row.get("variable"),
             old_value=row.get("old_value"),
             new_value=row.get("new_value"),
@@ -305,10 +389,11 @@ def load_evolution_xlsx(xlsx_path: Path) -> list[EvolutionEntry]:
                 entity=str(row[2]) if row[2] else "",
                 entity_id=str(row[3]) if row[3] else "",
                 parent_entity_id=str(row[4]) if row[4] else None,
-                variable=str(row[5]) if row[5] else None,
-                old_value=row[6] if row[6] else None,
-                new_value=row[7] if row[7] else None,
-                name=str(row[8]) if row[8] else None,
+                parent_entity=str(row[5]) if row[5] else None,
+                variable=str(row[6]) if row[6] else None,
+                old_value=row[7] if row[7] else None,
+                new_value=row[8] if row[8] else None,
+                name=str(row[9]) if len(row) > 9 and row[9] else None,
             )
         )
     return entries
@@ -345,6 +430,7 @@ def save_evolution(
         "entity",
         "entity_id",
         "parent_entity_id",
+        "parent_entity",
         "variable",
         "old_value",
         "new_value",
@@ -358,6 +444,7 @@ def save_evolution(
             entry.entity,
             entry.entity_id,
             entry.parent_entity_id,
+            entry.parent_entity,
             entry.variable,
             entry.old_value,
             entry.new_value,
@@ -390,6 +477,7 @@ def write_evolution_xlsx(entries: list[EvolutionEntry], xlsx_path: Path) -> None
         "entity",
         "entity_id",
         "parent_entity_id",
+        "parent_entity",
         "variable",
         "old_value",
         "new_value",
@@ -406,6 +494,7 @@ def write_evolution_xlsx(entries: list[EvolutionEntry], xlsx_path: Path) -> None
                 entry.entity,
                 str(entry.entity_id) if entry.entity_id is not None else "",
                 str(entry.parent_entity_id) if entry.parent_entity_id else "",
+                str(entry.parent_entity) if entry.parent_entity else "",
                 str(entry.variable) if entry.variable else "",
                 str(entry.old_value) if entry.old_value is not None else "",
                 str(entry.new_value) if entry.new_value is not None else "",

{jsonjsdb-0.7.3 → jsonjsdb-0.8.0}/src/jsonjsdb/writer.py

@@ -34,21 +34,26 @@ def write_table_jsonjs(df: pl.DataFrame, table_name: str, path: Path) -> None:
 
 
 def write_table_index(
-    tables: list[str], path: Path, timestamp: Optional[int] = None
+    tables: list[str],
+    path: Path,
+    timestamp: Optional[int] = None,
+    *,
+    write_js: bool = True,
 ) -> None:
-    """Write __table__.json with table metadata.
+    """Write __table__.json and optionally __table__.json.js with table metadata.
 
     Args:
         tables: List of table names to include
         path: Path to write __table__.json
         timestamp: Optional timestamp override (uses current time if None)
+        write_js: If True, also write __table__.json.js (default: True)
     """
     now = timestamp if timestamp is not None else int(time.time())
-    entries = [{"name": name, "last_modif": now} for name in sorted(tables)]
+    df = pl.DataFrame([{"name": name, "last_modif": now} for name in sorted(tables)])
 
-    with open(path, "w") as f:
-        json.dump(entries, f, indent=2, ensure_ascii=False)
-        f.write("\n")
+    write_table_json(df, path)
+    if write_js:
+        write_table_jsonjs(df, "__table__", path.with_suffix(".json.js"))
 
 
 def _prepare_df_for_write(df: pl.DataFrame) -> pl.DataFrame: