sibi-flux 2026.1.9__tar.gz → 2026.1.10__tar.gz

{sibi_flux-2026.1.9 → sibi_flux-2026.1.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sibi-flux
-Version: 2026.1.9
+Version: 2026.1.10
 Summary: Sibi Toolkit: A collection of tools for Data Analysis/Engineering.
 Author: Luis Valverde
 Author-email: Luis Valverde <lvalverdeb@gmail.com>

{sibi_flux-2026.1.9 → sibi_flux-2026.1.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sibi-flux"
-version = "2026.1.9"
+version = "2026.1.10"
 description = "Sibi Toolkit: A collection of tools for Data Analysis/Engineering."
 readme = "README.md"
 authors = [

{sibi_flux-2026.1.9 → sibi_flux-2026.1.10}/src/sibi_flux/datacube/cli.py

@@ -1057,20 +1057,31 @@ def scan(
 
     # Resolve global output file
     # First check nested discovery block (standard), then root (fallback)
-    discovery_conf = params.get("discovery", {})
+    # Resolve global output file
+    # First check nested discovery block (standard), then root (fallback)
+    discovery_conf = params.get("paths", {}).get("discovery") or params.get("discovery", {})
     global_tables_file = (
         discovery_conf.get("all_tables_file") 
         or params.get("all_tables_file") 
-        or "all_tables.yaml"
+        or "dataobjects/globals/all_tables.yaml"
     )
     
     # If using absolute path (resolved from context), use it directly
-    # Otherwise treat as relative to config file location
+    # Otherwise treat as relative to PROJECT ROOT (not config file location)
     path_obj = Path(global_tables_file)
     if path_obj.is_absolute():
         global_tables_path = path_obj
     else:
-        global_tables_path = config_path.parent / global_tables_file
+        try:
+             # Heuristic: config is in generators/datacubes
+             project_root = config_path.parent.parent.parent
+        except Exception:
+             project_root = Path.cwd()
+        global_tables_path = project_root / global_tables_file
+        
+        # Ensure dir exists
+        if not global_tables_path.parent.exists():
+            global_tables_path.parent.mkdir(parents=True, exist_ok=True)
 
     # Load existing data to preserve config for DBs not being scanned
     all_tables_data = {}
@@ -1245,6 +1256,8 @@ def propose_rules(
     dry_run: bool = typer.Option(
         False, "--dry-run", help="Preview rules without saving"
     ),
+    conf_override: Optional[str] = typer.Option(None, "--rules", help="Override rules file"),
+    force: bool = typer.Option(False, "--force", "-f", help="Force overwrite existing rules"),
 ):
     """
     Analyzes all_tables.yaml and proposes new discovery rules.
@@ -1261,23 +1274,35 @@ def propose_rules(
     # The resolved config will have absolute paths for these if the helper worked.
     params = context.params
 
-    # Prefer resolved values if available
-    if (
-        "discovery" in resolved_config
-        and "all_tables_file" in resolved_config["discovery"]
-    ):
-        all_tables_path = Path(resolved_config["discovery"]["all_tables_file"])
+    # Prefer resolved values if available, but check nested 'paths' structure first
+    # Structure: paths -> discovery -> all_tables_file
+    discovery_conf = params.get("paths", {}).get("discovery") or params.get("discovery", {})
+    all_tables_file = (
+        discovery_conf.get("all_tables_file")
+        or params.get("all_tables_file")
+        or "dataobjects/globals/all_tables.yaml"
+    )
+
+    path_obj = Path(all_tables_file)
+    if path_obj.is_absolute():
+        all_tables_path = path_obj
     else:
-        # Fallback to manual resolution (legacy or if not in discovery block)
-        raw_val = (
-            params.get("discovery", {}).get("all_tables_file")
-            or params.get("all_tables_file")
-            or "all_tables.yaml"
-        )
-        all_tables_path = config_path.parent / raw_val
+        try:
+             # Heuristic: config is in generators/datacubes
+             project_root = config_path.parent.parent.parent
+        except Exception:
+             project_root = Path.cwd()
+        all_tables_path = project_root / all_tables_file
+
+    # Verify Existence
+    if not all_tables_path.exists():
+         console.print(f"[red]Error: {all_tables_path} not found. Run 'dc scan' first.[/red]")
+         raise typer.Exit(code=1)
 
     # Rules File
-    if "discovery" in resolved_config and "rules_file" in resolved_config["discovery"]:
+    if conf_override:
+        rules_path = Path(conf_override)
+    elif "discovery" in resolved_config and "rules_file" in resolved_config["discovery"]:
         rules_path = Path(resolved_config["discovery"]["rules_file"])
     else:
         raw_rules = (
@@ -1287,11 +1312,12 @@ def propose_rules(
         )
         rules_path = config_path.parent / raw_rules
 
-    if not all_tables_path.exists():
-        console.print(
-            f"[red]Error: {all_tables_path} not found. Run 'dc-scan' first.[/red]"
-        )
-        raise typer.Exit(code=1)
+    # Guard: Require Force if Rules Exist
+    if rules_path.exists() and not force and not dry_run:
+        console.print(f"[yellow]Rules file exists at {rules_path}.[/yellow]")
+        console.print("[yellow]Skipping proposal generation to protect manual edits.[/yellow]")
+        console.print("[dim]Use --force to overwrite/update rules.[/dim]")
+        return
 
     engine = RuleEngine(all_tables_path, rules_path)
     engine.load()
@@ -1596,7 +1622,7 @@ def workflow(
     # Step 0: Scan & Propose Rules (Critical for new projects/drift)
     console.print(f"\n[bold cyan]Step 0a: Scan (Introspection)[/]")
     scan(
-        config_file=config_file,
+        config_file=config_file if config_file else None,
         db_url_map=db_url_map,
         env_file=env_file,
         db_name=db_name,
@@ -1604,8 +1630,10 @@ def workflow(
 
     console.print(f"\n[bold cyan]Step 0b: Propose Rules[/]")
     propose_rules(
-        config_file=config_file,
-        dry_run=dry_run,  # Preview rules if dry-running
+        config_file=config_file if config_file else None,
+        dry_run=dry_run,
+        conf_override=None,
+        force=force,
     )
 
     console.print(f"\n[bold cyan]Step 0c: Generate Whitelist[/]")
@@ -1936,8 +1964,38 @@ def whitelist(
             )
             continue
 
-        insp = inspect(engine)
-        tables = insp.get_table_names()
+        # 0. Try to load tables from all_tables.yaml (Scan Output)
+        tables = []
+        all_tables_file = (
+            params.get("paths", {}).get("discovery", {}).get("all_tables_file")
+            or params.get("all_tables_file")
+            or "dataobjects/globals/all_tables.yaml"
+        )
+        # Resolve path
+        if not Path(all_tables_file).is_absolute():
+            try:
+                prj_root = config_path.parent.parent.parent
+            except Exception:
+                prj_root = Path.cwd()
+            all_tables_path = prj_root / all_tables_file
+        else:
+            all_tables_path = Path(all_tables_file)
+
+        if all_tables_path.exists():
+            try:
+                with open(all_tables_path, "r") as f:
+                    all_tables_data = yaml.safe_load(f) or {}
+                    tables = all_tables_data.get(conn_obj) or []
+            except Exception:
+                pass
+
+        if tables:
+             console.print(f"[green]Loaded {len(tables)} tables from {all_tables_file} for {conn_obj}.[/green]")
+        else:
+            # Fallback to Live Introspection
+            console.print(f"[yellow]Warning: {conn_obj} not found in all_tables.yaml (or file missing). Falling back to live DB introspection.[/yellow]")
+            insp = inspect(engine)
+            tables = insp.get_table_names()
 
         # Determine path (Config Driven)
         discovery_cfg = params.get("paths", {}).get("discovery") or params.get(
@@ -2123,25 +2181,19 @@ def whitelist(
         # Construct new table map
         new_table_map = {}
 
-        # 1. Retained: Merge existing with new rule metadata
-        # Priority: Existing (Manual) > New (Rule)
-        # BUT: Enforce calculated paths to avoid stale absolute paths
         for t in retained:
-            existing_meta = existing_tables_map[t]
+            existing_meta = existing_tables_map.get(t, {})
             rule_meta = filtered_tables[t]
-            # Merge: update rule defaults only if not set in existing
-            merged = rule_meta.copy()
-            merged.update(existing_meta)  # Existing overwrites rule
             
-            # Legacy Cleanup: We moved custom_name to Registry
-            merged.pop("custom_name", None)
-
-            # Restore calculated paths (Enforce Relative)
-            if "datacube_path" in rule_meta:
-                merged["datacube_path"] = rule_meta["datacube_path"]
-            if "field_map_path" in rule_meta:
-                merged["field_map_path"] = rule_meta["field_map_path"]
-
+            # Strict Rewrite Logic (User Request)
+            # We ONLY preserve 'custom_name' from existing entries.
+            # All other metadata (domain, template, paths) must come strictly from rules/calculation.
+            custom_name = existing_meta.get("custom_name")
+            
+            merged = rule_meta.copy()
+            if custom_name:
+                merged["custom_name"] = custom_name
+                
             new_table_map[t] = merged
 
         # 2. Add new

{sibi_flux-2026.1.9 → sibi_flux-2026.1.10}/src/sibi_flux/datacube/config_engine.py

@@ -89,10 +89,16 @@ class ConfigurationEngine:
                         # Context not found and no default key
                         rules = []
 
+        # Sort Rules by Specificity (Length Descending)
+        # This prevents short prefixes (e.g. 'data_') from shadowing longer ones or exact matches if they overlap
+        # though usually exact matches don't overlap with delimiter-based prefixes.
+        # But for 'user_' vs 'use_', 'user_' (len 5) should come before 'use_' (len 4).
+        sorted_rules = sorted(rules, key=lambda x: len(x.get("pattern", "")), reverse=True)
+
         return GeneratorConfig(
             version=1.2,
             settings=GlobalSettings(**settings_data),
-            discovery_rules=[DiscoveryRule(**r) for r in rules],
+            discovery_rules=[DiscoveryRule(**r) for r in sorted_rules],
         )
 
     def _resolve_cubes_root(self) -> str:
@@ -213,6 +219,8 @@ class ConfigurationEngine:
             "field_map": field_map_module,  # 'field_map' template logic relies on discovery result often
             "connection_obj": conn_obj,
             "class_name": class_name,
+            # Pass template through for downstream consumers (e.g. whitelist generator)
+            "output_template": rule.output_template,
         }