sibi-flux 2026.1.8__py3-none-any.whl → 2026.1.9__py3-none-any.whl

sibi_flux/datacube/cli.py

@@ -203,7 +203,7 @@ def set_context_defaults(
 
 
 def _get_db_url_callback(
-    registry: DatacubeRegistry, db_url_map: Optional[str]
+    registry: DatacubeRegistry, db_url_map: Optional[str], params: Optional[Dict[str, Any]] = None
 ) -> Callable[[str], str]:
     """Helper to create a callback that resolves DB URLs from CLI overrides or registry."""
     cli_urls = json.loads(db_url_map) if db_url_map else {}
@@ -212,8 +212,24 @@ def _get_db_url_callback(
         # 1. CLI Override
         if conf_name in cli_urls:
             return cli_urls[conf_name]
+        
         # 2. Dynamic Resolution
-        url = resolve_db_url(conf_name, registry.global_imports)
+        imports = registry.global_imports
+        
+        # Check specific import_spec from params
+        if params and "databases" in params:
+             for db in params.get("databases", []):
+                  ref = db.get("connection_ref") or db.get("connection_obj")
+                  if ref == conf_name:
+                       spec = db.get("import_spec")
+                       if spec and isinstance(spec, dict):
+                            # Prepend specific module
+                            imports = [spec.get("module")] + imports
+                       elif db.get("global_import"):
+                            imports = [db.get("global_import")] + imports
+                       break
+        
+        url = resolve_db_url(conf_name, imports)
         if url:
             return url
         raise ValueError(
@@ -283,9 +299,16 @@ def sync(
                 
                 flat_tables = {}
                 for grp, tbls in existing_reg_data.items():
-                    if isinstance(tbls, dict):
+                    if grp == "tables" and isinstance(tbls, dict):
+                        # Modern format: tables are in 'tables' dict
+                        for t, t_meta in tbls.items():
+                            flat_tables[t] = t_meta
+                        continue
+
+                    if isinstance(tbls, dict) and grp != "global_imports":
+                        # Legacy Scoped Format (grp is connection_obj)
                         for t, t_meta in tbls.items():
-                            # Inject the config object (group key) so DatacubeRegistry knows the connection
+                            # Inject/Overwrite connection_obj only if implied by scope
                             t_meta["connection_obj"] = grp
                             flat_tables[t] = t_meta
                 
@@ -295,6 +318,11 @@ def sync(
                 console.print(f"[yellow]Warning: Could not load existing registry: {e}[/yellow]")
 
     registry = DatacubeRegistry(config_data, params=context.params)
+    
+    # Fix 2 Re-applied: Ensure global_imports are populated from params if registry loaded from file didn't have them
+    # This is critical for Clean Project + Force flows where registry might be partial or JIT
+    if not registry.global_imports and context.params.get("global_imports"):
+        registry.global_imports = context.params.get("global_imports")
 
     # --- Aggregation Phase ---
     # --- Aggregation Phase ---
@@ -453,7 +481,7 @@ def sync(
     registry.valid_paths = list(valid_paths)
     registry.valid_fieldmap_paths = context.valid_fieldmap_paths
 
-    get_url = _get_db_url_callback(registry, db_url_map)
+    get_url = _get_db_url_callback(registry, db_url_map, params=context.params)
 
     # Group tables by target file
     file_groups = registry.group_tables_by_file()
@@ -770,8 +798,22 @@ def discover(
         if not wl_filename:
             # Default convention: discovery_whitelist_<db_name>.yaml
             wl_filename = f"discovery_whitelist_{conn_obj}.yaml"
+        
         whitelist_path = config_path.parent / wl_filename
 
+        # Fallback 1: Auto-generated per-db (from scan/map fallback logic)
+        if not whitelist_path.exists():
+            fb = config_path.parent / f"discovery_whitelist_{conn_obj}.yaml"
+            if fb.exists():
+                whitelist_path = fb
+        
+        # Fallback 2: Global whitelist.yaml (Generated by 'whitelist' command)
+        if not whitelist_path.exists():
+            global_wl = config_path.parent / "whitelist.yaml"
+            if global_wl.exists():
+                console.print(f"[dim]Fallback: Using global {global_wl.name} for {db_name}[/dim]")
+                whitelist_path = global_wl
+
         # Determine rules path
         rules_filename = db_config.get("rules_file")
         if not rules_filename:
@@ -1102,7 +1144,7 @@ def drift(
     config_data = _load_and_resolve_config(config_path)
 
     registry = DatacubeRegistry(config_data)
-    get_url = _get_db_url_callback(registry, db_url_map)
+    get_url = _get_db_url_callback(registry, db_url_map, params=config_data)
     cli_urls = json.loads(db_url_map) if db_url_map else {}
 
     drift_table = Table(title="Schema Drift Analysis")
@@ -1566,6 +1608,15 @@ def workflow(
         dry_run=dry_run,  # Preview rules if dry-running
     )
 
+    console.print(f"\n[bold cyan]Step 0c: Generate Whitelist[/]")
+    whitelist(
+        config_file=config_file,
+        db_name=db_name,
+        db_url_map=db_url_map,
+        env_file=env_file,
+        force=force,
+    )
+
     # Step 1: Discover
     # Note: Discover currently runs for all DBs in config.
     console.print(f"\n[bold cyan]Step 1: Discovery[/]")
@@ -1582,17 +1633,10 @@ def workflow(
         force=False, # Discovery force is about fields usually?
     )
 
-    # Step 2: Map
-    console.print(f"\n[bold cyan]Step 2: Map Generation[/]")
-    map(
-        config_file=config_file,
-        db_url_map=db_url_map,
-        env_file=env_file,
-        db_name=db_name,
-        force=force,
-    )
-
-    # Step 3: Sync
+    # Step 2: Map (Skipped - Handled by Sync)
+    # console.print(f"\n[bold cyan]Step 2: Map Generation (Integrated into Sync)[/]")
+    
+    # Step 3: Sync -> Generates Code + Field Maps
     console.print(f"\n[bold cyan]Step 3: Sync (Code Generation)[/]")
     sync(
         config_file=config_file,

sibi_flux/datacube/generator.py

@@ -778,6 +778,32 @@ def generate_datacube_module_code(
         else:
             imports.add(base_imp)
 
+        # Config Object Import
+        # We need to resolve import for conf_obj (e.g. clickhouse_conf)
+        # Using registry.params which contains databases list with import_spec
+        params = registry.params
+        dbs = params.get("databases", [])
+        for db in dbs:
+            ref = db.get("connection_ref") or db.get("connection_obj")
+            if ref == conf_obj:
+               spec = db.get("import_spec")
+               if spec and isinstance(spec, dict):
+                   mod = spec.get("module")
+                   sym = spec.get("symbol")
+                   if mod and sym:
+                       imports.add(f"from {mod} import {sym}")
+               elif db.get("global_import"):
+                   # Legacy or simple string import
+                   imp_str = db.get("global_import")
+                   if imp_str and sym and sym in imp_str:
+                        imports.add(imp_str)
+                   elif imp_str: # Fallback to add specific object if simple import
+                       # Try to parse or just add "from ... import conf_obj" logic?
+                       # If global_import is "from conf.credentials import *" -> hard to know
+                       # If it is "from conf.credentials import replica_conf" -> easy
+                       imports.add(imp_str)
+               break
+
         details = registry.get_table_details(table_name)
         field_map_str = details.get("field_map")
         sticky_filters = details.get("sticky_filters")

sibi_flux/init/cube_proposer.py

@@ -56,6 +56,8 @@ def propose_cubes(db_domain: str, app_name: str) -> None:
     
     # Registry Structure: {conf_obj: {table_name: {class_name: ..., path: ...}}}
     for conf_obj, tables in global_data.items():
+        if not isinstance(tables, dict):
+             continue
         for table_name, meta in tables.items():
             path_str = meta.get("path", "")
             class_name = meta.get("class_name")
@@ -100,14 +102,16 @@ def propose_cubes(db_domain: str, app_name: str) -> None:
     
     # 4. Update App Registry
     
-    # Load existing
     current_config = {"cubes": []}
     if registry_path.exists():
         try:
             with open(registry_path, "r") as f:
                 loaded = yaml.safe_load(f)
-                if loaded and "cubes" in loaded:
-                    current_config = loaded
+                if loaded:
+                     current_config = loaded
+                     if "cubes" not in current_config or current_config["cubes"] is None:
+                         current_config["cubes"] = []
+    
         except Exception:
             pass
             

{sibi_flux-2026.1.8.dist-info → sibi_flux-2026.1.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sibi-flux
-Version: 2026.1.8
+Version: 2026.1.9
 Summary: Sibi Toolkit: A collection of tools for Data Analysis/Engineering.
 Author: Luis Valverde
 Author-email: Luis Valverde <lvalverdeb@gmail.com>

{sibi_flux-2026.1.8.dist-info → sibi_flux-2026.1.9.dist-info}/RECORD

@@ -23,12 +23,12 @@ sibi_flux/dask_cluster/exceptions.py,sha256=apQZaUMgac8k2ZTTsvUd-VlWdo9-Nrh5b6St
 sibi_flux/dask_cluster/utils.py,sha256=Pr2qaow6GVyvM0hqKSM0ZQpe2Ot5ayfGYQiNhNpYA8Y,1342
 sibi_flux/datacube/__init__.py,sha256=ODEoa4r5RtzynIp-PdVDaJ-4BcPBj1L9VkLIF7RYSPE,91
 sibi_flux/datacube/_data_cube.py,sha256=Ofgy3JlR7N0eKijpUzI-ixFlISUd3CFsxnKd6a4fguE,12629
-sibi_flux/datacube/cli.py,sha256=LoEa1lyKyCclAFZ4OEC9rFy6cyTtGoQ55xgSEQzMMDM,101107
+sibi_flux/datacube/cli.py,sha256=z3NxwkGd44v58X3pxsOWcvYyvXOW_4_61V-gD_Kkzqs,103410
 sibi_flux/datacube/config_engine.py,sha256=3cmxycCqMzjYJ1fWn8IWrHTNwSQmRqxnPoOzc1lUtDk,8340
 sibi_flux/datacube/field_factory.py,sha256=Z3Yp6tGzrZ13rvKSjMFr9jvW7fazeNi4K1fAalxLujM,6376
 sibi_flux/datacube/field_mapper.py,sha256=V6aFYunl28DI7gSvrF7tcidPNX9QtOYymVxbumzQqPs,9334
 sibi_flux/datacube/field_registry.py,sha256=VBTqxNIn2-eWMiZi1oQK-719oyJfn3yhfV7Bz_0iNyU,4355
-sibi_flux/datacube/generator.py,sha256=gPGQW114N7jfyGwsSXc-jsJup_C4qSj75oM_CoWuP7Y,36943
+sibi_flux/datacube/generator.py,sha256=oDkJozogn_wf-jrOPvtgtw_8rew9AviNQIM1MbSFQa0,38274
 sibi_flux/datacube/orchestrator.py,sha256=ZS2kh1Q6FF1sPtDaTkIvo8Q2S33OrYogjOWDl15-u-k,25254
 sibi_flux/datacube/router.py,sha256=SdG0J4O9CSZeUkHjyWHFC8B3oH83fSdDHfMXZfmHfi0,10914
 sibi_flux/dataset/__init__.py,sha256=zYpf4-nalcQNSUIOCDG0CqQu2SKkm89I9AF6Zy6z1sE,53
@@ -68,7 +68,7 @@ sibi_flux/init/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 sibi_flux/init/app.py,sha256=Ofx_Sowvv6Ri-tDhLYkYllB_e1-4227J-MqhURTAmhI,3678
 sibi_flux/init/core.py,sha256=ahl4wVAyOrl1ko-f2NcYxeX9yQBQceYnobqeH_Gfp98,7785
 sibi_flux/init/cube_extender.py,sha256=wQPM0zncr97AiGCVM2BYo7dyaLKWIndV-0GvGNXvAMA,5269
-sibi_flux/init/cube_proposer.py,sha256=VA7WF-pDtto9-UdN1IlB7u6TiuxgMbgy79TRooyBPro,5071
+sibi_flux/init/cube_proposer.py,sha256=FdcSbdgV7z2a2omeCJAhlM7zcnix440oMAz1bzl6LlE,5242
 sibi_flux/init/discovery_updater.py,sha256=3WONU50EnaIItaLncumd3hC6h059ykvoqMfRC5NAvi8,5652
 sibi_flux/init/env.py,sha256=WMczkJqdaCejSbkjcpW8a44GZdKicdh5GVfUqKYltBI,3792
 sibi_flux/init/env_engine.py,sha256=HYBik80qgp-_2RgnQY2edaKA30VY1k_3uAZ4TqIID2M,6247
@@ -123,7 +123,7 @@ sibi_flux/utils/file_utils.py,sha256=7OHUW65OTe6HlQ6wkDagDd7d0SCQ_-NEGmHlOJguKYw
 sibi_flux/utils/filepath_generator/__init__.py,sha256=YVFJhIewjwksb9E2t43ojNC-W_AqUDhkKxQVBIBMkY8,91
 sibi_flux/utils/filepath_generator/_filepath_generator.py,sha256=4HG-Ubvjtv6luL0z-A-8B6_r3o9YqBwATFXhOXiTbKc,6789
 sibi_flux/utils/retry.py,sha256=45t0MF2IoMayN9xkn5_FtakMq4HwZlGvHVd6qv8x1AY,1227
-sibi_flux-2026.1.8.dist-info/WHEEL,sha256=ZyFSCYkV2BrxH6-HRVRg3R9Fo7MALzer9KiPYqNxSbo,79
-sibi_flux-2026.1.8.dist-info/entry_points.txt,sha256=6xrq5zuz_8wodJj4s49raopnuC3Owy_leZRkWtcXpTk,49
-sibi_flux-2026.1.8.dist-info/METADATA,sha256=Jm2z9nxpK77TbqwAxWvA_s8fy5y_N71LJzfNnAwkLFM,9886
-sibi_flux-2026.1.8.dist-info/RECORD,,
+sibi_flux-2026.1.9.dist-info/WHEEL,sha256=ZyFSCYkV2BrxH6-HRVRg3R9Fo7MALzer9KiPYqNxSbo,79
+sibi_flux-2026.1.9.dist-info/entry_points.txt,sha256=6xrq5zuz_8wodJj4s49raopnuC3Owy_leZRkWtcXpTk,49
+sibi_flux-2026.1.9.dist-info/METADATA,sha256=vaMygw_hUTc3ORvTh4kYbt_5v5Ovvtdm3dchFZxOGF8,9886
+sibi_flux-2026.1.9.dist-info/RECORD,,