sibi-flux 2026.1.6__tar.gz → 2026.1.8__tar.gz

{sibi_flux-2026.1.6 → sibi_flux-2026.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sibi-flux
-Version: 2026.1.6
+Version: 2026.1.8
 Summary: Sibi Toolkit: A collection of tools for Data Analysis/Engineering.
 Author: Luis Valverde
 Author-email: Luis Valverde <lvalverdeb@gmail.com>

{sibi_flux-2026.1.6 → sibi_flux-2026.1.8}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sibi-flux"
-version = "2026.1.6"
+version = "2026.1.8"
 description = "Sibi Toolkit: A collection of tools for Data Analysis/Engineering."
 readme = "README.md"
 authors = [

{sibi_flux-2026.1.6 → sibi_flux-2026.1.8}/src/sibi_flux/datacube/cli.py

@@ -478,6 +478,35 @@ def sync(
         is_append = False
         existing_content = ""
 
+        # --- Registry Collection (Always run, even if skipped) ---
+        # Structure: {conf_obj: {table_name: {class_name: ..., path: ...}}}
+        for item in items:
+            t_name = item[0]
+            conf_obj = item[1]
+            cls_n = item[4]
+            # Calculate path relative to project root
+            try:
+                if "project_root" not in locals():
+                    project_root = config_path.parent.parent.parent
+                rel_path = file_path.relative_to(project_root)
+            except Exception:
+                rel_path = file_path
+
+            if conf_obj not in generated_registry:
+                generated_registry[conf_obj] = {}
+
+            # Preserve custom_name from existing registry if present
+            existing_meta = registry.get_table_details(t_name)
+            custom_name = existing_meta.get("custom_name")
+
+            entry_data = {
+                "class_name": cls_n,
+                "path": str(rel_path),
+                "custom_name": custom_name,
+            }
+
+            generated_registry[conf_obj][t_name] = entry_data
+
         if file_path.exists() and not force:
             with open(file_path, "r") as f:
                 existing_content = f.read()
@@ -524,12 +553,8 @@ def sync(
         )
 
         if not classes_code:
-            if not is_append:
-                summary_table.add_row(
-                    file_path_str, "0", "[red]Failed (No Classes Generated)[/red]"
-                )
-            else:
-                summary_table.add_row(file_path_str, "0", "[red]Failed to Append[/red]")
+            status_msg = "[red]Failed (No Classes Generated)[/red]" if not is_append else "[red]Failed to Append[/red]"
+            summary_table.add_row(file_path_str, "0", status_msg)
             continue
 
         if not is_append:
@@ -558,36 +583,6 @@ def sync(
         )
         summary_table.add_row(file_path_str, str(len(items)), status_msg)
 
-        # --- Registry Collection ---
-        # Collect metadata for generated datacubes
-        # Structure: {conf_obj: {table_name: {class_name: ..., path: ...}}}
-        for item in items:
-            t_name = item[0]
-            conf_obj = item[1]
-            cls_n = item[4]
-            # Calculate path relative to project root
-            try:
-                if "project_root" not in locals():
-                    project_root = config_path.parent.parent.parent
-                rel_path = file_path.relative_to(project_root)
-            except Exception:
-                rel_path = file_path
-
-            if conf_obj not in generated_registry:
-                generated_registry[conf_obj] = {}
-
-            # Preserve custom_name from existing registry if present
-            existing_meta = registry.get_table_details(t_name)
-            custom_name = existing_meta.get("custom_name")
-
-            entry_data = {
-                "class_name": cls_n,
-                "path": str(rel_path),
-                "custom_name": custom_name,
-            }
-
-            generated_registry[conf_obj][t_name] = entry_data
-
     console.print(summary_table)
 
     # --- Write Datacube Registry ---
@@ -616,8 +611,9 @@ def sync(
             with open(reg_file, "w") as f:
                 yaml.dump(reg_data, f, sort_keys=False)
 
+            total_tables = sum(len(tables) for tables in generated_registry.values())
             console.print(
-                f"[green]Updated Datacube Registry at {reg_rel_path} ({len(generated_registry)} entries)[/green]"
+                f"[green]Updated Datacube Registry at {reg_rel_path} ({total_tables} tables across {len(generated_registry)} groups)[/green]"
             )
         except Exception as e:
             console.print(f"[red]Failed to write Datacube Registry: {e}[/red]")
@@ -815,12 +811,29 @@ def discover(
             )
             continue
 
+        # Resolve Registry Path (Target)
+        reg_rel_path = params.get("paths", {}).get("repositories", {}).get(
+            "global_datacube_registry_file"
+        ) or params.get("global_datacube_registry_file")
+        
+        real_registry_path = str(config_path) # Fallback to config if not defined (legacy behavior)
+        if reg_rel_path:
+             if Path(reg_rel_path).is_absolute():
+                 real_registry_path = reg_rel_path
+             else:
+                 try:
+                     # Anchor to project root
+                     prj_root = config_path.parent.parent.parent
+                     real_registry_path = str(prj_root / reg_rel_path)
+                 except Exception:
+                     real_registry_path = str(config_path.parent / reg_rel_path)
+
         orchestrator = DiscoveryOrchestrator(
             field_registry=field_registry,
             params=context.params,
             rules_path=str(rules_path),
             whitelist_path=str(whitelist_path),
-            registry_path=str(config_path),
+            registry_path=real_registry_path,
             db_connection_str=db_conn_str,
             db_config=db_config,
         )
@@ -1001,8 +1014,21 @@ def scan(
             raise typer.Exit(code=1)
 
     # Resolve global output file
-    global_tables_file = params.get("all_tables_file") or "all_tables.yaml"
-    global_tables_path = config_path.parent / global_tables_file
+    # First check nested discovery block (standard), then root (fallback)
+    discovery_conf = params.get("discovery", {})
+    global_tables_file = (
+        discovery_conf.get("all_tables_file") 
+        or params.get("all_tables_file") 
+        or "all_tables.yaml"
+    )
+    
+    # If using absolute path (resolved from context), use it directly
+    # Otherwise treat as relative to config file location
+    path_obj = Path(global_tables_file)
+    if path_obj.is_absolute():
+        global_tables_path = path_obj
+    else:
+        global_tables_path = config_path.parent / global_tables_file
 
     # Load existing data to preserve config for DBs not being scanned
     all_tables_data = {}
@@ -1480,6 +1506,114 @@ def map(
     return
 
 
+@app.command()
+def workflow(
+    config_file: Optional[Path] = typer.Option(None, "--config"),
+    db_url_map: Optional[str] = typer.Option(None, "--db-urls"),
+    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e"),
+    db_name: Optional[str] = typer.Option(
+        None, "--db", help="Target specific database (filtering map/sync)"
+    ),
+    app_name: Optional[str] = typer.Option(
+        None, "--app-name", help="Run create_cubes for the specified application"
+    ),
+    force: bool = typer.Option(False, "--force", "-f", help="Force rebuild/overwrite"),
+    dry_run: bool = typer.Option(
+        False, "--dry-run", help="Preview changes without executing"
+    ),
+) -> None:
+    """
+    Executes the full Datacube workflow: Scan -> Propose -> Discover -> Map -> Sync -> (Optional) Create Cubes.
+    """
+    console.rule("[bold magenta]Starting Datacube Workflow[/]")
+
+    # Resolve env_file: CLI > Params > Default (Consistent with discover/sync)
+    if env_file:
+        env_path = env_file
+    elif context.params and context.params.get("defaults", {}).get("env_file"):
+        env_path = Path(context.params.get("defaults", {})["env_file"])
+    else:
+        env_path = Path(".env.linux")
+    load_environment(env_path, logger=console.print)
+
+    # Ensure context is configured with params from the config file
+    config_path = config_file or context.default_config
+    if config_path and config_path.exists():
+        # Load and resolve config to ensure context.params is populated
+        # This is critical for 'discover' to find 'databases' list
+        try:
+            config_data = _load_and_resolve_config(config_path)
+            context.params = config_data
+            context.default_config = config_path
+            # We don't have easy access to other set_context_defaults args here, 
+            # but params is what 'discover' needs most.
+            console.print(f"[dim]Loaded workflow context from {config_path}[/dim]")
+        except Exception as e:
+            console.print(f"[yellow]Warning: Failed to load context from {config_path}: {e}[/yellow]")
+
+    # Step 0: Scan & Propose Rules (Critical for new projects/drift)
+    console.print(f"\n[bold cyan]Step 0a: Scan (Introspection)[/]")
+    scan(
+        config_file=config_file,
+        db_url_map=db_url_map,
+        env_file=env_file,
+        db_name=db_name,
+    )
+
+    console.print(f"\n[bold cyan]Step 0b: Propose Rules[/]")
+    propose_rules(
+        config_file=config_file,
+        dry_run=dry_run,  # Preview rules if dry-running
+    )
+
+    # Step 1: Discover
+    # Note: Discover currently runs for all DBs in config.
+    console.print(f"\n[bold cyan]Step 1: Discovery[/]")
+    discover(
+        config_file=config_file,
+        db_conf="replica_db_conf", # Pass explicit string to avoid Typer OptionInfo default
+        db_url_map=db_url_map,
+        env_file=env_file,
+        update=True,  # Always update registry to enable chaining
+        prune=False,  # Safer default? Or should we expose prune?
+        run_sync=False, # We handle sync explicitly later
+        dry_run=dry_run,
+        generate_fields=False, # We use map command for this
+        force=False, # Discovery force is about fields usually?
+    )
+
+    # Step 2: Map
+    console.print(f"\n[bold cyan]Step 2: Map Generation[/]")
+    map(
+        config_file=config_file,
+        db_url_map=db_url_map,
+        env_file=env_file,
+        db_name=db_name,
+        force=force,
+    )
+
+    # Step 3: Sync
+    console.print(f"\n[bold cyan]Step 3: Sync (Code Generation)[/]")
+    sync(
+        config_file=config_file,
+        db_url_map=db_url_map,
+        env_file=env_file, # sync command has env_file
+        force=force,
+        dry_run=dry_run,
+    )
+
+    # Step 4: Create Cubes (Optional)
+    if app_name:
+        console.print(f"\n[bold cyan]Step 4: Application Cubes ({app_name})[/]")
+        if dry_run:
+            console.print("[yellow]Skipping create_cubes in dry-run mode.[/yellow]")
+        else:
+            from sibi_flux.init.cube_extender import create_cubes
+            create_cubes(app_name)
+
+    console.rule("[bold magenta]Workflow Completed[/]")
+
+
 @app.command()
 def init(
     config_file: Optional[Path] = typer.Option(None, "--config"),