sibi-flux 2026.1.5__tar.gz → 2026.1.7__tar.gz

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sibi-flux
-Version: 2026.1.5
+Version: 2026.1.7
 Summary: Sibi Toolkit: A collection of tools for Data Analysis/Engineering.
 Author: Luis Valverde
 Author-email: Luis Valverde <lvalverdeb@gmail.com>
@@ -52,30 +52,10 @@ Description-Content-Type: text/markdown
 
 **SibiFlux** is a production-grade resilient data engineering ecosystem designed to bridge the gap between local development, distributed computing, and agentic AI workflows. It provides a unified engine for hybrid data loading (batch + streaming), self-healing distributed operations, and native interfaces for AI agents via the Model Context Protocol (MCP).
 
-```mermaid
-graph TD
-    subgraph G1["Agentic Interface (MCP)"]
-        Agent["AI Agent / Claude"] <--> Router["MCP Router"]
-        Router <--> Resources["SibiFlux Resources"]
-    end
-
-    subgraph G2["Solutions Layer (Business Logic)"]
-        Logistics["Logistics Solutions"]
-        Enrichment["Enrichment Pipelines"]
-        Cubes["DataCubes"]
-    end
-
-    subgraph G3["SibiFlux Core Engine"]
-        DfHelper["DfHelper (Unified Loader)"]
-        Cluster["Resilient Dask Cluster"]
-        Managed["ManagedResource Lifecycle"]
-    end
-
-    Resources --> Cubes
-    Logistics --> DfHelper
-    Cubes --> DfHelper
-    DfHelper --> Cluster
-```
+
+
+## Documentation
+Full documentation is available in [src/docs/index.md](src/docs/index.md).
 
 ## Core Architecture
 

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/README.md

@@ -2,30 +2,10 @@
 
 **SibiFlux** is a production-grade resilient data engineering ecosystem designed to bridge the gap between local development, distributed computing, and agentic AI workflows. It provides a unified engine for hybrid data loading (batch + streaming), self-healing distributed operations, and native interfaces for AI agents via the Model Context Protocol (MCP).
 
-```mermaid
-graph TD
-    subgraph G1["Agentic Interface (MCP)"]
-        Agent["AI Agent / Claude"] <--> Router["MCP Router"]
-        Router <--> Resources["SibiFlux Resources"]
-    end
-
-    subgraph G2["Solutions Layer (Business Logic)"]
-        Logistics["Logistics Solutions"]
-        Enrichment["Enrichment Pipelines"]
-        Cubes["DataCubes"]
-    end
-
-    subgraph G3["SibiFlux Core Engine"]
-        DfHelper["DfHelper (Unified Loader)"]
-        Cluster["Resilient Dask Cluster"]
-        Managed["ManagedResource Lifecycle"]
-    end
-
-    Resources --> Cubes
-    Logistics --> DfHelper
-    Cubes --> DfHelper
-    DfHelper --> Cluster
-```
+
+
+## Documentation
+Full documentation is available in [src/docs/index.md](src/docs/index.md).
 
 ## Core Architecture
 

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sibi-flux"
-version = "2026.1.5"
+version = "2026.1.7"
 description = "Sibi Toolkit: A collection of tools for Data Analysis/Engineering."
 readme = "README.md"
 authors = [
@@ -127,6 +127,21 @@ dc-scan = "python solutions/generators/datacubes/gen_dc.py scan"
 dc-match = "python solutions/generators/datacubes/gen_dc.py match"
 dc-map = "python solutions/generators/datacubes/gen_dc.py map"
 
+# --- Internal Snapshot Tasks ---
+_test_snapshots_core = "python tests/solutions/gendatacubes/test_snapshots.py"
+_test_snapshots_cli = "python tests/solutions/gendatacubes/test_cli_snapshot.py"
+_test_snapshots_env = "python tests/solutions/init/test_env_generator_snapshots.py"
+_test_snapshots_rules = "python tests/solutions/init/test_rule_engine_snapshots.py"
+
+[tool.poe.tasks.test-snapshots]
+help = "Run all regression snapshot tests"
+sequence = ["_test_snapshots_core", "_test_snapshots_cli", "_test_snapshots_env", "_test_snapshots_rules"]
+
+[tool.poe.tasks.update-snapshots]
+help = "Update all regression snapshots (Golden Masters)"
+sequence = ["_test_snapshots_core", "_test_snapshots_cli", "_test_snapshots_env", "_test_snapshots_rules"]
+env = { UPDATE_SNAPSHOTS = "1" }
+
 [tool.poe.tasks.release]
 sequence = ["build","publish"]
 envfile = ".env"  # Loads the token for the whole sequence

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/src/sibi_flux/cli.py

@@ -34,6 +34,42 @@ def init(
     initialize_project(project_name, lib, app)
 
 
+@app.command()
+def create_app(
+    name: str = typer.Argument(..., help="Name of the application to create"),
+):
+    """
+    Create a new application within an existing Sibi Flux project.
+    
+    Generates standard directory structure in `<project_root>/<name>`.
+    """
+    from sibi_flux.init.app import init_app
+    init_app(name)
+
+
+@app.command()
+def create_cubes(
+    app_name: str = typer.Argument(..., help="Name of the application"),
+):
+    """
+    Generate app-specific Datacube extensions from `<app_name>/datacubes/datacubes.yaml`.
+    """
+    from sibi_flux.init.cube_extender import create_cubes
+    create_cubes(app_name)
+
+
+@app.command()
+def propose_cubes(
+    db_domain: str = typer.Argument(..., help="Database domain to filter by (e.g., 'ibis_dev')"),
+    app_name: str = typer.Argument(..., help="Name of the target application"),
+):
+    """
+    Scan global registry for datacubes in <db_domain> and add them to <app_name>/datacubes/datacubes.yaml.
+    """
+    from sibi_flux.init.cube_proposer import propose_cubes
+    propose_cubes(db_domain, app_name)
+
+
 @app.command()
 def env(
     project_path: Path = typer.Argument(Path("."), help="Project root directory"),

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/src/sibi_flux/datacube/cli.py

@@ -258,6 +258,42 @@ def sync(
 
     # Start with empty/default registry
     config_data = _load_and_resolve_config(config_path)
+
+    # Load existing Global Registry to preserve manual edits (e.g. custom_name)
+    # We must flatten the grouped structure (by config object) into a single tables dict
+    params = context.params or {}
+    reg_rel_path = params.get("paths", {}).get("repositories", {}).get(
+        "global_datacube_registry_file"
+    ) or params.get("global_datacube_registry_file")
+
+    if reg_rel_path:
+        reg_file = Path(reg_rel_path)
+        if not reg_file.is_absolute():
+            try:
+                # Heuristic: config_path is in generators/datacubes/, project root is 3 levels up
+                prj_root = config_path.parent.parent.parent
+                reg_file = prj_root / reg_rel_path
+            except Exception:
+                reg_file = Path.cwd() / reg_rel_path
+        
+        if reg_file.exists():
+            try:
+                with open(reg_file, "r") as rf:
+                    existing_reg_data = yaml.safe_load(rf) or {}
+                
+                flat_tables = {}
+                for grp, tbls in existing_reg_data.items():
+                    if isinstance(tbls, dict):
+                        for t, t_meta in tbls.items():
+                            # Inject the config object (group key) so DatacubeRegistry knows the connection
+                            t_meta["connection_obj"] = grp
+                            flat_tables[t] = t_meta
+                
+                config_data["tables"] = flat_tables
+                console.print(f"[dim]Loaded {len(flat_tables)} existing registry entries for merge preservation.[/dim]")
+            except Exception as e:
+                console.print(f"[yellow]Warning: Could not load existing registry: {e}[/yellow]")
+
     registry = DatacubeRegistry(config_data, params=context.params)
 
     # --- Aggregation Phase ---
@@ -378,9 +414,16 @@ def sync(
             imp = [db_imp] if db_imp else registry.global_imports
             return resolve_db_url(conf_name, imp)
 
-        _run_field_map_generation(
+        generated_maps = _run_field_map_generation(
             context, config_path, databases, get_url_safe, force=force
         )
+
+        if generated_maps:
+            for table_name, mod_path in generated_maps.items():
+                if table_name in registry.tables:
+                    # console.print(f"[green]DEBUG: Updating {table_name} with field_map {mod_path}[/green]")
+                    registry.tables[table_name]["field_map"] = mod_path
+
         # Ensure new modules are picked up
         importlib.invalidate_caches()
 
@@ -435,6 +478,35 @@ def sync(
         is_append = False
         existing_content = ""
 
+        # --- Registry Collection (Always run, even if skipped) ---
+        # Structure: {conf_obj: {table_name: {class_name: ..., path: ...}}}
+        for item in items:
+            t_name = item[0]
+            conf_obj = item[1]
+            cls_n = item[4]
+            # Calculate path relative to project root
+            try:
+                if "project_root" not in locals():
+                    project_root = config_path.parent.parent.parent
+                rel_path = file_path.relative_to(project_root)
+            except Exception:
+                rel_path = file_path
+
+            if conf_obj not in generated_registry:
+                generated_registry[conf_obj] = {}
+
+            # Preserve custom_name from existing registry if present
+            existing_meta = registry.get_table_details(t_name)
+            custom_name = existing_meta.get("custom_name")
+
+            entry_data = {
+                "class_name": cls_n,
+                "path": str(rel_path),
+                "custom_name": custom_name,
+            }
+
+            generated_registry[conf_obj][t_name] = entry_data
+
         if file_path.exists() and not force:
             with open(file_path, "r") as f:
                 existing_content = f.read()
@@ -481,12 +553,8 @@ def sync(
         )
 
         if not classes_code:
-            if not is_append:
-                summary_table.add_row(
-                    file_path_str, "0", "[red]Failed (No Classes Generated)[/red]"
-                )
-            else:
-                summary_table.add_row(file_path_str, "0", "[red]Failed to Append[/red]")
+            status_msg = "[red]Failed (No Classes Generated)[/red]" if not is_append else "[red]Failed to Append[/red]"
+            summary_table.add_row(file_path_str, "0", status_msg)
             continue
 
         if not is_append:
@@ -515,29 +583,6 @@ def sync(
         )
         summary_table.add_row(file_path_str, str(len(items)), status_msg)
 
-        # --- Registry Collection ---
-        # Collect metadata for generated datacubes
-        # Structure: {conf_obj: {table_name: {class_name: ..., path: ...}}}
-        for item in items:
-            t_name = item[0]
-            conf_obj = item[1]
-            cls_n = item[4]
-            # Calculate path relative to project root
-            try:
-                if "project_root" not in locals():
-                    project_root = config_path.parent.parent.parent
-                rel_path = file_path.relative_to(project_root)
-            except Exception:
-                rel_path = file_path
-
-            if conf_obj not in generated_registry:
-                generated_registry[conf_obj] = {}
-
-            generated_registry[conf_obj][t_name] = {
-                "class_name": cls_n,
-                "path": str(rel_path),
-            }
-
     console.print(summary_table)
 
     # --- Write Datacube Registry ---
@@ -566,8 +611,9 @@ def sync(
             with open(reg_file, "w") as f:
                 yaml.dump(reg_data, f, sort_keys=False)
 
+            total_tables = sum(len(tables) for tables in generated_registry.values())
             console.print(
-                f"[green]Updated Datacube Registry at {reg_rel_path} ({len(generated_registry)} entries)[/green]"
+                f"[green]Updated Datacube Registry at {reg_rel_path} ({total_tables} tables across {len(generated_registry)} groups)[/green]"
             )
         except Exception as e:
             console.print(f"[red]Failed to write Datacube Registry: {e}[/red]")
@@ -765,12 +811,29 @@ def discover(
             )
             continue
 
+        # Resolve Registry Path (Target)
+        reg_rel_path = params.get("paths", {}).get("repositories", {}).get(
+            "global_datacube_registry_file"
+        ) or params.get("global_datacube_registry_file")
+        
+        real_registry_path = str(config_path) # Fallback to config if not defined (legacy behavior)
+        if reg_rel_path:
+             if Path(reg_rel_path).is_absolute():
+                 real_registry_path = reg_rel_path
+             else:
+                 try:
+                     # Anchor to project root
+                     prj_root = config_path.parent.parent.parent
+                     real_registry_path = str(prj_root / reg_rel_path)
+                 except Exception:
+                     real_registry_path = str(config_path.parent / reg_rel_path)
+
         orchestrator = DiscoveryOrchestrator(
             field_registry=field_registry,
             params=context.params,
             rules_path=str(rules_path),
             whitelist_path=str(whitelist_path),
-            registry_path=str(config_path),
+            registry_path=real_registry_path,
             db_connection_str=db_conn_str,
             db_config=db_config,
         )
@@ -1904,6 +1967,9 @@ def whitelist(
             # Merge: update rule defaults only if not set in existing
             merged = rule_meta.copy()
             merged.update(existing_meta)  # Existing overwrites rule
+            
+            # Legacy Cleanup: We moved custom_name to Registry
+            merged.pop("custom_name", None)
 
             # Restore calculated paths (Enforce Relative)
             if "datacube_path" in rule_meta:
@@ -2099,6 +2165,8 @@ def _run_field_map_generation(
                 "[yellow]Warning: Could not determine global field_maps_dir for clean build.[/yellow]"
             )
 
+    generated_maps = {}
+    
     for db in target_dbs:
         # console.print(f"DEBUG: Processing DB entry: {db} (Type: {type(db)})")
         if isinstance(db, str):
@@ -2223,10 +2291,6 @@ def _run_field_map_generation(
             if not found:
                 rules = []
 
-        # console.print(f"DEBUG: Loaded {len(rules)} rules from {rules_path} for {conn_obj}")
-
-        # console.print(f"DEBUG: Loaded {len(rules)} rules from {rules_path}")
-
         # Support List or Dict Format
         scoped_data = registry_data.get(conn_obj, {})
         if isinstance(scoped_data, list):
@@ -2448,6 +2512,21 @@ def _run_field_map_generation(
                     with open(target_file, "w") as f:
                         f.write("\n".join(lines))
 
+                    # Calculate Import Path
+                    try:
+                        # Ensure we get relative path to project root (which should be sys.path root)
+                        if "project_root" not in locals():
+                            project_root = Path.cwd()
+                        
+                        rel_py_path = target_file.relative_to(project_root)
+                        module_path = str(rel_py_path.with_suffix("")).replace("/", ".")
+                        generated_maps[table_name] = f"{module_path}.field_map"
+                    except ValueError:
+                        # Fallback if outside project root?
+                        pass
+                    except Exception as e:
+                        pass
+
                 except Exception as e:
                     console.print(f"[red]Error processing {table_name}: {e}[/red]")
                     continue
@@ -2464,6 +2543,8 @@ def _run_field_map_generation(
     except Exception as e:
         console.print(f"[red]Failed to save Global Field Repository: {e}[/red]")
 
+    return generated_maps
+
 
 if __name__ == "__main__":
     app()

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/src/sibi_flux/datacube/generator.py

@@ -531,6 +531,15 @@ class DatacubeRegistry:
             or self.config.get("class_suffix")
             or self.params.get("class_suffix", "Dc")
         )
+        self._enforce_custom_names()
+
+    def _enforce_custom_names(self) -> None:
+        """
+        Ensures that if custom_name is set, it overrides class_name explicitly.
+        """
+        for table, meta in self.tables.items():
+            if meta.get("custom_name"):
+                meta["class_name"] = meta["custom_name"]
 
     def get_table_details(self, table_name: str) -> dict[str, Any]:
         return self.tables.get(table_name, {})
@@ -573,7 +582,10 @@ class DatacubeRegistry:
                     elif k not in existing:
                         existing[k] = v
 
-                if "class_name" not in existing:
+                # Override class_name if custom_name is set
+                if existing.get("custom_name"):
+                    existing["class_name"] = existing["custom_name"]
+                elif "class_name" not in existing:
                     existing["class_name"] = new_details.get("class_name")
 
                 self.tables[table] = existing

{sibi_flux-2026.1.5 → sibi_flux-2026.1.7}/src/sibi_flux/datacube/orchestrator.py

@@ -554,8 +554,23 @@ class DiscoveryOrchestrator:
                 "[yellow]Prune active: Registry replaced with discovery results.[/]"
             )
         else:
-            # Merge: Update existing keys, add new ones.
-            current_data["tables"].update(new_entries)
+            # Smart Merge: Preserve 'custom_name' and 'class_name' from existing entries
+            # if they are not explicitly overridden by the new entry.
+            for table, new_meta in new_entries.items():
+                if table in current_data["tables"]:
+                    existing = current_data["tables"][table]
+                    
+                    # 1. Preserve custom_name if new entry doesn't specify one
+                    if not new_meta.get("custom_name") and existing.get("custom_name"):
+                        new_meta["custom_name"] = existing["custom_name"]
+                        
+                        # 2. Also preserve class_name (since it's driven by custom_name)
+                        # We only preserve class_name if custom_name was preserved 
+                        # AND new entry didn't explicitly change class_name logic (unlikely unless configured)
+                        if existing.get("class_name"):
+                            new_meta["class_name"] = existing["class_name"]
+
+                current_data["tables"][table] = new_meta
 
         # Sort tables for readability
         current_data["tables"] = dict(sorted(current_data["tables"].items()))

sibi_flux-2026.1.7/src/sibi_flux/init/app.py

@@ -0,0 +1,111 @@
+
+from pathlib import Path
+import os
+from rich.console import Console
+import typer
+import yaml
+
+console = Console()
+
+def init_app(name: str) -> None:
+    """
+    Initialize a new application within the current project root.
+    
+    Args:
+        name: The name of the application to create (e.g., 'inventory', 'pricing').
+    """
+    
+    # 1. Validation: Ensure we are in a Sibi Flux project root (check for pyproject.toml as heuristic)
+    cwd = Path(os.getcwd())
+    if not (cwd / "pyproject.toml").exists():
+         console.print("[yellow]Warning: pyproject.toml not found. Are you in a project root?[/yellow]")
+    
+    app_dir = cwd / name
+    
+    if app_dir.exists():
+        console.print(f"[red]Error: Application directory '{name}' already exists.[/red]")
+        raise typer.Exit(code=1)
+        
+    console.print(f"[bold blue]Initializing new application: {name}[/bold blue]")
+    
+    # 2. Create Directory Structure
+    structure = [
+        "api",
+        "datacubes",
+        "readers",
+        "aggregators",
+    ]
+    
+    app_dir.mkdir()
+    (app_dir / "__init__.py").touch()
+    
+    for folder in structure:
+        path = app_dir / folder
+        path.mkdir()
+        (path / "__init__.py").touch()
+        
+    # 2.1 Create datacubes extension registry template
+    datacubes_yaml = app_dir / "datacubes" / "datacubes.yaml"
+    template_yaml = """
+cubes:
+  # Define your app-specific datacube extensions here
+  # - source: dataobjects.gencubes.ibis_dev.products.products_cubes.ProductsDc
+  #   name: LogisticsProductsDc
+  #   module: products  # -> logistics/datacubes/products.py
+"""
+    datacubes_yaml.write_text(template_yaml.strip())
+        
+    # 3. Create Basic Router Template
+    router_template = f"""
+from fastapi import APIRouter
+
+router = APIRouter()
+
+@router.get("/")
+async def root():
+    return {{"message": "Hello from {name}!"}}
+"""
+    (app_dir / "api" / "main.py").write_text(router_template.strip())
+    
+    # 4. Register in conf/apps.yaml
+    conf_dir = cwd / "conf"
+    if not conf_dir.exists():
+        conf_dir.mkdir()
+    
+    apps_yaml_path = conf_dir / "apps.yaml"
+    
+    apps_data = {"apps": []}
+    if apps_yaml_path.exists():
+        try:
+            with open(apps_yaml_path, "r") as f:
+                loaded = yaml.safe_load(f)
+                if loaded and isinstance(loaded, dict) and "apps" in loaded:
+                    apps_data = loaded
+                elif loaded is None:
+                     pass # empty file
+                else:
+                    console.print(f"[yellow]Warning: conf/apps.yaml has unexpected structure. Initializing with empty list.[/yellow]")
+        except Exception as e:
+            console.print(f"[yellow]Warning: Could not read existing apps.yaml: {e}[/yellow]")
+    
+    if name not in apps_data["apps"]:
+        apps_data["apps"].append(name)
+        with open(apps_yaml_path, "w") as f:
+            yaml.dump(apps_data, f, default_flow_style=False)
+        console.print(f"[green]Registered '{name}' in conf/apps.yaml[/green]")
+    else:
+         console.print(f"[yellow]App '{name}' already registered in conf/apps.yaml[/yellow]")
+
+    # 5. Success Message & Instructions
+    console.print(f"[bold green]Successfully created application '{name}'![/bold green]")
+    console.print(f"Location: {name}/")
+    console.print("\n[yellow]Next Steps:[/yellow]")
+    console.print(f"1. Register your new router in [bold]main.py[/bold] (or wherever your app is defined):")
+    
+    code_snippet = f"""
+    from {name}.api.main import router as {name}_router
+    
+    app.include_router({name}_router, prefix="/{name}", tags=["{name}"])
+    """
+    console.print(code_snippet)
+