sibi-flux 2026.1.1__tar.gz → 2026.1.2__tar.gz

{sibi_flux-2026.1.1 → sibi_flux-2026.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sibi-flux
-Version: 2026.1.1
+Version: 2026.1.2
 Summary: Sibi Toolkit: A collection of tools for Data Analysis/Engineering.
 Author: Luis Valverde
 Author-email: Luis Valverde <lvalverdeb@gmail.com>
@@ -28,6 +28,7 @@ Requires-Dist: opentelemetry-api>=1.38.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.38.0
 Requires-Dist: opentelemetry-sdk>=1.38.0
 Requires-Dist: deep-translator>=1.11.4
+Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: sibi-flux[distributed,geospatial,mcp] ; extra == 'complete'
 Requires-Dist: distributed>=2025.11.0 ; extra == 'distributed'
 Requires-Dist: osmnx>=2.0.7 ; extra == 'geospatial'
@@ -282,3 +283,43 @@ async with GenericMcpClient(url="http://localhost:8000/sse") as client:
     # Call a tool
     vat = await client.call_tool("calculate_vat", arguments={"amount": 100.0})
 ```
+
+### 8. Datacube Generation (`gen_dc.py`)
+
+Automate the creation of Datacube classes and Field Maps from your database schema.
+
+**Configuration (`discovery_params.yaml`)**
+
+Define your generation rules in a hierarchical configuration file:
+
+```yaml
+defaults:
+  backend: sqlalchemy
+  class_suffix: Dc
+
+discovery:
+  whitelist_file: whitelist.yaml
+  rules_file: discovery_rules.yaml
+
+generation:
+  enable_field_maps: true
+```
+
+**Commands**
+
+1.  **Discover**: Introspects the database and updates your whitelist/registry.
+    ```bash
+    uv run poe dc-discover
+    ```
+    *   **Whitelist**: Explicitly define tables to generate. Support `custom_name` to override class names.
+    *   **Rules**: Regex-based patterns to match tables.
+
+2.  **Sync**: Generates Python code (Datacubes and Field Maps) based on the registry.
+    ```bash
+    uv run poe dc-sync --force
+    ```
+
+**Key Features**
+- **Custom Naming**: Add `custom_name: MyCube` to `whitelist.yaml` to override generated names.
+- **Hierarchical Config**: Strict validation of generation parameters.
+- **Field Maps**: Auto-generates type-safe mapping files for every table.

{sibi_flux-2026.1.1 → sibi_flux-2026.1.2}/README.md

@@ -231,3 +231,43 @@ async with GenericMcpClient(url="http://localhost:8000/sse") as client:
     # Call a tool
     vat = await client.call_tool("calculate_vat", arguments={"amount": 100.0})
 ```
+
+### 8. Datacube Generation (`gen_dc.py`)
+
+Automate the creation of Datacube classes and Field Maps from your database schema.
+
+**Configuration (`discovery_params.yaml`)**
+
+Define your generation rules in a hierarchical configuration file:
+
+```yaml
+defaults:
+  backend: sqlalchemy
+  class_suffix: Dc
+
+discovery:
+  whitelist_file: whitelist.yaml
+  rules_file: discovery_rules.yaml
+
+generation:
+  enable_field_maps: true
+```
+
+**Commands**
+
+1.  **Discover**: Introspects the database and updates your whitelist/registry.
+    ```bash
+    uv run poe dc-discover
+    ```
+    *   **Whitelist**: Explicitly define tables to generate. Support `custom_name` to override class names.
+    *   **Rules**: Regex-based patterns to match tables.
+
+2.  **Sync**: Generates Python code (Datacubes and Field Maps) based on the registry.
+    ```bash
+    uv run poe dc-sync --force
+    ```
+
+**Key Features**
+- **Custom Naming**: Add `custom_name: MyCube` to `whitelist.yaml` to override generated names.
+- **Hierarchical Config**: Strict validation of generation parameters.
+- **Field Maps**: Auto-generates type-safe mapping files for every table.

{sibi_flux-2026.1.1 → sibi_flux-2026.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sibi-flux"
-version = "2026.1.1"
+version = "2026.1.2"
 description = "Sibi Toolkit: A collection of tools for Data Analysis/Engineering."
 readme = "README.md"
 authors = [
@@ -34,6 +34,7 @@ dependencies = [
     "opentelemetry-exporter-otlp>=1.38.0",
     "opentelemetry-sdk>=1.38.0",
     "deep-translator>=1.11.4",
+    "pyyaml>=6.0.3",
 ]
 
 [project.optional-dependencies]
@@ -60,6 +61,9 @@ complete = [
     "sibi-flux[distributed,geospatial,mcp]"
 ]
 
+[project.scripts]
+sibi-flux = "sibi_flux.cli:app"
+
 
 [dependency-groups]
 dev = [
@@ -90,6 +94,35 @@ build-backend = "uv_build"
 module-root = "src"
 module-name = ["sibi_flux", "sibi_dst"]
 
+[tool.uv.workspace]
+members = [
+    "test-project-alpha",
+    "test-project-beta",
+    "test-project-gamma",
+    "test-project-delta",
+    "test-project-epsilon",
+    "test-project-zeta",
+    "test-project-eta",
+    "test-project-theta",
+    "test-project-iota",
+    "test-project-kappa",
+    "test-project-lambda",
+    "test-project-alpha/latest",
+    "test-project-mu",
+    "test-project-mu/latest",
+    "test-project-xi",
+    "test-project-xi/latest",
+    "test-project-omicron",
+    "test-project-omicron/latest",
+    "test-project-pi",
+    "test-project-pi/latest",
+    "test-project-rho",
+    "test-project-rho/latest",
+    "test-project-sigma",
+    "test-project-sigma/latest",
+    "test-project-tau",
+]
+
 [tool.pytest.ini_options]
 pythonpath = ["src", "."]
 testpaths = ["tests"]

sibi_flux-2026.1.2/src/sibi_flux/cli.py

@@ -0,0 +1,45 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from rich.console import Console
+from sibi_flux.init.core import initialize_project
+
+app = typer.Typer(help="Sibi Flux CLI")
+console = Console()
+
+@app.callback()
+def callback():
+    """
+    Sibi Flux CLI
+    """
+
+@app.command()
+def init(
+    project_name: str = typer.Argument(..., help="Name of the project to create"),
+    lib: bool = typer.Option(False, "--lib", help="Initialize as a library project (passed to uv init)"),
+    app: bool = typer.Option(False, "--app", help="Initialize as an application project (passed to uv init)")
+):
+    """
+    Initialize a new Sibi Flux project.
+    
+    Creates a new directory <project_name>, initializes it with 'uv',
+    and adds 'sibi-flux' as a dependency.
+    """
+    initialize_project(project_name, lib, app)
+
+@app.command()
+def env(
+    project_path: Path = typer.Argument(Path("."), help="Project root directory"),
+    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e", help="Path to environment file (defaults to .env)"),
+    cleanup: bool = typer.Option(False, "--cleanup", help="Remove existing configuration files"),
+    production: bool = typer.Option(False, "--production", "-p", help="Generate production skeleton (no hardcoded values)"),
+):
+    """
+    Initialize configuration files (settings.py, credentials) based on .env
+    """
+    from sibi_flux.init.env import init_env
+    init_env(project_path, env_file, cleanup=cleanup, production_mode=production)
+
+
+if __name__ == "__main__":
+    app()

{sibi_flux-2026.1.1 → sibi_flux-2026.1.2}/src/sibi_flux/config/settings.py

@@ -1,4 +1,4 @@
-from typing import Optional, Any
+from typing import Optional, Any, ClassVar
 from pydantic import SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -9,6 +9,8 @@ class SibiBaseSettings(BaseSettings):
     model_config = SettingsConfigDict(
         env_file=".env", env_file_encoding="utf-8", extra="ignore"
     )
+    
+    conf_name: ClassVar[str] = ""
 
 
 class FsSettings(SibiBaseSettings):
@@ -84,13 +86,8 @@ class DatabaseSettings(SibiBaseSettings):
     """Generic SQL Database settings."""
 
     db_url: str = "sqlite:///:memory:"
-
-
-class ClickhouseBaseSettings(SibiBaseSettings):
-    """Base settings for ClickHouse connection."""
-
     host: str = "localhost"
-    port: int = 8123
+    port: int = 5432
     database: str = "default"
     user: str = "default"
     password: SecretStr = SecretStr("secret")
@@ -102,9 +99,12 @@ class ClickhouseBaseSettings(SibiBaseSettings):
             "dbname": self.database,
             "user": self.user,
             "password": self.password.get_secret_value() if self.password else None,
+            "db_url": self.db_url,
         }
 
 
+
+
 class RedisBaseSettings(SibiBaseSettings):
     """Base settings for Redis connection."""
 

sibi_flux-2026.1.2/src/sibi_flux/init/core.py

@@ -0,0 +1,159 @@
+import subprocess
+import os
+import sys
+from pathlib import Path
+from rich.console import Console
+import typer
+
+console = Console()
+
+def initialize_project(
+    project_name: str,
+    lib: bool = False,
+    app: bool = False,
+) -> None:
+    """
+    Initialize a new Sibi Flux project with the standard directory structure and configuration.
+
+    Args:
+        project_name: The name of the project directory to create.
+        lib: If True, initialize as a library project via `uv init --lib`.
+        app: If True, initialize as an application project via `uv init --app`.
+             Note: If `lib` is False, the default behavior will be to include `--app` 
+             unless `app` is explicitly False. (Aligned with CLI defaults).
+    """
+    project_path = Path(os.getcwd()) / project_name
+    
+    if project_path.exists():
+        console.print(f"[red]Error: Directory '{project_name}' already exists.[/red]")
+        raise typer.Exit(code=1)
+    
+    console.print(f"[bold blue]Initializing Sibi Flux project: {project_name}[/bold blue]")
+    
+    # 1. Create Directory
+    try:
+        # Run `uv init`
+        # format: uv init <project_name> [--lib] [--app]
+        cmd = ["uv", "init", project_name]
+        
+        if lib:
+            cmd.append("--lib")
+        else:
+            # Default behavior: behave like --app unless explicitly disabled?
+            # From previous steps, user wanted default to be --app.
+            # If lib is False, we default to app.
+            cmd.append("--app")
+            
+        console.print(f"Running: {' '.join(cmd)}")
+        subprocess.check_call(cmd)
+        
+        # 2. Add sibi-flux dependency
+        console.print(f"Adding sibi-flux dependency...")
+        try:
+            from importlib.metadata import version
+            ver = version("sibi-flux")
+            pkg_spec = f"sibi-flux>={ver}"
+        except Exception:
+            pkg_spec = "sibi-flux@latest"
+            
+        subprocess.check_call(["uv", "add", pkg_spec], cwd=project_path)
+
+        # 2b. Add dev dependencies
+        dev_deps = ["black", "notebook", "pytest", "ruff", "httpx", "poethepoet", "PyYAML"]
+        console.print(f"Adding dev dependencies: {', '.join(dev_deps)}...")
+        subprocess.check_call(["uv", "add", "--dev"] + dev_deps, cwd=project_path)
+        
+        # 3. Create Scaffolding Folders (conf, dataobjects, generators)
+        (project_path / "conf").mkdir(exist_ok=True)
+        (project_path / "dataobjects").mkdir(exist_ok=True)
+        (project_path / "generators" / "datacubes").mkdir(parents=True, exist_ok=True)
+        
+        # Add basic .gitkeep
+        (project_path / "conf" / ".gitkeep").touch()
+        (project_path / "dataobjects" / ".gitkeep").touch()
+        (project_path / "generators" / ".gitkeep").touch()
+        # (project_path / "generators" / "datacubes" / ".gitkeep").touch() # Not needed if we write params
+        
+        # 4. Write discovery_params.yaml templates
+        try:
+             from importlib import resources as importlib_resources
+             pass
+        except ImportError:
+             import importlib_resources # type: ignore
+
+        # Access the template resource from sibi_flux.init.templates
+        try:
+            # Modern way
+            ref = importlib_resources.files("sibi_flux.init.templates").joinpath("discovery_params.yaml")
+            template_content = ref.read_text()
+        except AttributeError:
+            # Python < 3.9
+            template_content = importlib_resources.read_text("sibi_flux.init.templates", "discovery_params.yaml")
+
+        # Refactor paths for the new project structure
+        template_content = template_content.replace("solutions/dataobjects/", "dataobjects/")
+        template_content = template_content.replace("solutions.conf", "conf")
+        template_content = template_content.replace(".env.linux", ".env.local")
+
+        (project_path / "generators" / "datacubes" / "discovery_params.yaml").write_text(template_content)
+
+        # Write property_template.yaml
+        try:
+            ref_prop = importlib_resources.files("sibi_flux.init.templates").joinpath("property_template.yaml")
+            prop_content = ref_prop.read_text()
+        except AttributeError:
+            prop_content = importlib_resources.read_text("sibi_flux.init.templates", "property_template.yaml")
+
+        (project_path / "generators" / "datacubes" / "property_template.yaml").write_text(prop_content)
+
+        # Write gen_dc.py
+        try:
+            ref_gen = importlib_resources.files("sibi_flux.init.templates").joinpath("gen_dc.py")
+            gen_content = ref_gen.read_text()
+        except AttributeError:
+            gen_content = importlib_resources.read_text("sibi_flux.init.templates", "gen_dc.py")
+
+        (project_path / "generators" / "datacubes" / "gen_dc.py").write_text(gen_content)
+
+        # 5. Create .env.local
+        (project_path / ".env.local").touch()
+
+        # 6. Create/Update .gitignore
+        gitignore_path = project_path / ".gitignore"
+        gitignore_content = ""
+        if gitignore_path.exists():
+            gitignore_content = gitignore_path.read_text()
+        
+        # Ensure .env is ignored
+        if ".env" not in gitignore_content:
+            with open(gitignore_path, "a") as f:
+                if gitignore_content and not gitignore_content.endswith("\n"):
+                    f.write("\n")
+                f.write("\n# Environment variables\n.env\n.env.local\n.env.*\n")
+        
+        if not gitignore_path.exists():
+             gitignore_path.write_text("# Sibi Flux\n.env\n.env.local\n.env.*\n__pycache__/\n*.pyc\n.DS_Store\n")
+
+        # 7. Configure poe tasks in pyproject.toml
+        pyproject_path = project_path / "pyproject.toml"
+        if pyproject_path.exists():
+            with open(pyproject_path, "a") as f:
+                f.write("\n\n[tool.poe.tasks]\n")
+                f.write('dc-sync = "python generators/datacubes/gen_dc.py sync"\n')
+                f.write('dc-init = "python generators/datacubes/gen_dc.py init"\n')
+                f.write('dc-discover = "python generators/datacubes/gen_dc.py discover"\n')
+                f.write('dc-scan = "python generators/datacubes/gen_dc.py scan"\n')
+                f.write('dc-match = "python generators/datacubes/gen_dc.py match"\n')
+                f.write('dc-map = "python generators/datacubes/gen_dc.py map"\n')
+
+        console.print(f"[bold green]Successfully initialized {project_name}![/bold green]")
+        console.print(f"Created directories: conf/, dataobjects/, generators/datacubes/")
+        console.print(f"Created files: .env.local, .gitignore, generators/datacubes/[discovery_params.yaml, property_template.yaml, gen_dc.py]")
+        console.print(f"cd {project_name}")
+        
+    except subprocess.CalledProcessError as e:
+        console.print(f"[red]Command failed with exit code {e.returncode}[/red]")
+        raise typer.Exit(code=e.returncode)
+    except FileNotFoundError:
+        console.print("[red]Error: 'uv' command not found. Please ensure uv is installed and in your PATH.[/red]")
+        raise typer.Exit(code=1)

sibi_flux-2026.1.2/src/sibi_flux/init/discovery_updater.py

@@ -0,0 +1,99 @@
+import yaml
+from pathlib import Path
+from typing import List, Dict, Any
+from sibi_flux.init.env_engine import EnvFile, EnvSection
+from rich.console import Console
+
+console = Console()
+
+class DiscoveryParamsUpdater:
+    """
+    Updates generators/datacubes/discovery_params.yaml to include
+    database connections found in the generated env configuration.
+    """
+
+    @staticmethod
+    def _is_database(section: EnvSection) -> bool:
+        return section.type.upper() in ["POSTGRES", "MYSQL", "CLICKHOUSE", "SQLALCHEMY_DATABASE"]
+
+    @staticmethod
+    def update(project_path: Path, env_data: EnvFile) -> None:
+        params_path = project_path / "generators" / "datacubes" / "discovery_params.yaml"
+        if not params_path.exists():
+            console.print(f"[yellow]Warning: {params_path} not found. Skipping discovery params update.[/yellow]")
+            return
+
+        try:
+            # 1. Read existing YAML
+            with open(params_path, "r") as f:
+                data = yaml.safe_load(f) or {}
+
+            # 2. Extract DBs from Env
+            db_sections = [s for s in env_data.sections if DiscoveryParamsUpdater._is_database(s)]
+            
+            if not db_sections:
+                if "databases" in data:
+                    del data["databases"]
+                    with open(params_path, "w") as f:
+                        yaml.dump(data, f, sort_keys=False, default_flow_style=False)
+                    console.print(f"[yellow]Removed databases section from discovery_params.yaml (no databases found in env).[/yellow]")
+                return
+
+            # Ensure databases list exists
+            if "databases" not in data:
+                data["databases"] = []
+            
+            # Map existing entries by ID to avoid duplicates (upsert behavior)
+            existing_dbs = {db.get("id"): db for db in data["databases"]}
+
+            changes_made = False
+
+            for section in db_sections:
+                db_id = section.name.lower()
+                
+                # Check exclusion? No, default to include all.
+                
+                # Generate entry
+                conf_name = f"{section.name.lower()}_conf"
+                if section.type.upper() == "CLICKHOUSE":
+                    conf_name = "clickhouse_config"
+                
+                # Extract DB Name for db_domain
+                db_domain = db_id # Fallback
+                for var in section.vars:
+                    # Check for common DB name keys. 
+                    # Note: Keys in EnvSection are fully qualified (e.g. REPLICA_DATABASE)
+                    # We check if the suffix matches DATABASE
+                    if var.key.endswith("_DATABASE") or var.key == "DATABASE":
+                         db_domain = var.value
+                         break
+
+                new_entry = {
+                    "id": db_id,
+                    "connection_ref": conf_name,
+                    "db_domain": db_domain,
+                    "import_spec": {
+                        "module": "conf.credentials",
+                        "symbol": conf_name
+                    }
+                }
+                
+                # Upsert
+                if db_id in existing_dbs:
+                    # Update existing? Only if significantly different? 
+                    # For now, let's update strict fields but preserve extra if any.
+                    # Or simpler: Overwrite specific keys.
+                    existing_dbs[db_id].update(new_entry)
+                else:
+                    data["databases"].append(new_entry)
+                    changes_made = True
+
+            # 3. Write back
+            # Use sort_keys=False to preserve order roughly
+            with open(params_path, "w") as f:
+                yaml.dump(data, f, sort_keys=False, default_flow_style=False)
+            
+            console.print(f"[green]Updated discovery_params.yaml with {len(db_sections)} databases.[/green]")
+
+        except Exception as e:
+             console.print(f"[red]Failed to update discovery_params.yaml: {e}[/red]")

sibi_flux-2026.1.2/src/sibi_flux/init/env.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+from rich.console import Console
+from typing import Optional
+
+from sibi_flux.init.env_engine import EnvParser
+from sibi_flux.init.env_generator import EnvGenerator
+
+console = Console()
+
+def init_env(project_path: Path, env_file: Optional[Path] = None, cleanup: bool = False, production_mode: bool = False):
+    """
+    Initializes configuration files for the project.
+    Parses .env file (or defaults) and dynamically generates settings.py and credentials.
+    If cleanup is True, removes generated configuration files.
+    If production_mode is True, generates a 'skeleton' settings.py without hardcoded values.
+    """
+    conf_dir = project_path / "conf"
+    
+    if cleanup:
+        settings_path = conf_dir / "settings.py"
+        creds_path = conf_dir / "credentials" / "__init__.py"
+        
+        removed = []
+        if settings_path.exists():
+            settings_path.unlink()
+            removed.append("conf/settings.py")
+        
+        if creds_path.exists():
+            creds_path.unlink()
+            removed.append("conf/credentials/__init__.py")
+            
+        if removed:
+            console.print(f"[green]Removed: {', '.join(removed)}[/green]")
+        else:
+            console.print("[yellow]No configuration files found to remove.[/yellow]")
+        return
+
+    # Parse Environment
+    if env_file:
+         if not env_file.exists():
+             console.print(f"[red]Error: Environment file {env_file} does not exist.[/red]")
+             return
+         
+         content = env_file.read_text().strip()
+         if not content or all(line.strip().startswith('#') for line in content.splitlines() if line.strip()):
+             console.print(f"[yellow]Environment file {env_file} is empty or contains only comments. No action taken.[/yellow]")
+             return
+             
+         env_data = EnvParser.parse(content)
+         env_filename = env_file.name
+    else:
+         # Implicit default: try to find .env, but if not found or empty, maybe generate skeleton?
+         # User request implies: "if an .env file is empty, message that nothing will be done"
+         # But if NO file passed, we assume .env in project root.
+         
+         default_env = project_path / ".env"
+         if not default_env.exists():
+              console.print(f"[yellow]No .env file found at {default_env}. Skipping generation.[/yellow]")
+              return
+
+         content = default_env.read_text().strip()
+         if not content or all(line.strip().startswith('#') for line in content.splitlines() if line.strip()):
+             console.print(f"[yellow]Environment file .env is empty or contains only comments. No action taken.[/yellow]")
+             return
+             
+         env_data = EnvParser.parse(content)
+         env_filename = ".env"
+
+    conf_dir.mkdir(exist_ok=True)
+    
+    # 1. settings.py
+    settings_code = EnvGenerator.generate_pydantic_code(env_data, env_file_name=env_filename, production_mode=production_mode)
+    (conf_dir / "settings.py").write_text(settings_code)
+    console.print(f"[green]Created conf/settings.py (parsed {len(env_data.sections)} sections)[/green]")
+
+    # 2. credentials/__init__.py
+    creds_dir = conf_dir / "credentials"
+    creds_dir.mkdir(exist_ok=True)
+    
+    creds_code = EnvGenerator.generate_credentials_code(env_data)
+    (creds_dir / "__init__.py").write_text(creds_code)
+    console.print(f"[green]Created conf/credentials/__init__.py[/green]")
+
+    # 3. Update discovery_params.yaml
+    from sibi_flux.init.discovery_updater import DiscoveryParamsUpdater
+    DiscoveryParamsUpdater.update(project_path, env_data)