sibi-flux 2026.1.1__py3-none-any.whl → 2026.1.2__py3-none-any.whl

sibi_flux/cli.py

@@ -0,0 +1,45 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from rich.console import Console
+from sibi_flux.init.core import initialize_project
+
+app = typer.Typer(help="Sibi Flux CLI")
+console = Console()
+
+@app.callback()
+def callback():
+    """
+    Sibi Flux CLI
+    """
+
+@app.command()
+def init(
+    project_name: str = typer.Argument(..., help="Name of the project to create"),
+    lib: bool = typer.Option(False, "--lib", help="Initialize as a library project (passed to uv init)"),
+    app: bool = typer.Option(False, "--app", help="Initialize as an application project (passed to uv init)")
+):
+    """
+    Initialize a new Sibi Flux project.
+    
+    Creates a new directory <project_name>, initializes it with 'uv',
+    and adds 'sibi-flux' as a dependency.
+    """
+    initialize_project(project_name, lib, app)
+
+@app.command()
+def env(
+    project_path: Path = typer.Argument(Path("."), help="Project root directory"),
+    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e", help="Path to environment file (defaults to .env)"),
+    cleanup: bool = typer.Option(False, "--cleanup", help="Remove existing configuration files"),
+    production: bool = typer.Option(False, "--production", "-p", help="Generate production skeleton (no hardcoded values)"),
+):
+    """
+    Initialize configuration files (settings.py, credentials) based on .env
+    """
+    from sibi_flux.init.env import init_env
+    init_env(project_path, env_file, cleanup=cleanup, production_mode=production)
+
+
+if __name__ == "__main__":
+    app()

sibi_flux/config/settings.py

@@ -1,4 +1,4 @@
-from typing import Optional, Any
+from typing import Optional, Any, ClassVar
 from pydantic import SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -9,6 +9,8 @@ class SibiBaseSettings(BaseSettings):
     model_config = SettingsConfigDict(
         env_file=".env", env_file_encoding="utf-8", extra="ignore"
     )
+    
+    conf_name: ClassVar[str] = ""
 
 
 class FsSettings(SibiBaseSettings):
@@ -84,13 +86,8 @@ class DatabaseSettings(SibiBaseSettings):
     """Generic SQL Database settings."""
 
     db_url: str = "sqlite:///:memory:"
-
-
-class ClickhouseBaseSettings(SibiBaseSettings):
-    """Base settings for ClickHouse connection."""
-
     host: str = "localhost"
-    port: int = 8123
+    port: int = 5432
     database: str = "default"
     user: str = "default"
     password: SecretStr = SecretStr("secret")
@@ -102,9 +99,12 @@ class ClickhouseBaseSettings(SibiBaseSettings):
             "dbname": self.database,
             "user": self.user,
             "password": self.password.get_secret_value() if self.password else None,
+            "db_url": self.db_url,
         }
 
 
+
+
 class RedisBaseSettings(SibiBaseSettings):
     """Base settings for Redis connection."""
 

sibi_flux/init/core.py

@@ -0,0 +1,159 @@
+import subprocess
+import os
+import sys
+from pathlib import Path
+from rich.console import Console
+import typer
+
+console = Console()
+
+def initialize_project(
+    project_name: str,
+    lib: bool = False,
+    app: bool = False,
+) -> None:
+    """
+    Initialize a new Sibi Flux project with the standard directory structure and configuration.
+
+    Args:
+        project_name: The name of the project directory to create.
+        lib: If True, initialize as a library project via `uv init --lib`.
+        app: If True, initialize as an application project via `uv init --app`.
+             Note: If `lib` is False, the default behavior will be to include `--app` 
+             unless `app` is explicitly False. (Aligned with CLI defaults).
+    """
+    project_path = Path(os.getcwd()) / project_name
+    
+    if project_path.exists():
+        console.print(f"[red]Error: Directory '{project_name}' already exists.[/red]")
+        raise typer.Exit(code=1)
+    
+    console.print(f"[bold blue]Initializing Sibi Flux project: {project_name}[/bold blue]")
+    
+    # 1. Create Directory
+    try:
+        # Run `uv init`
+        # format: uv init <project_name> [--lib] [--app]
+        cmd = ["uv", "init", project_name]
+        
+        if lib:
+            cmd.append("--lib")
+        else:
+            # Default behavior: behave like --app unless explicitly disabled?
+            # From previous steps, user wanted default to be --app.
+            # If lib is False, we default to app.
+            cmd.append("--app")
+            
+        console.print(f"Running: {' '.join(cmd)}")
+        subprocess.check_call(cmd)
+        
+        # 2. Add sibi-flux dependency
+        console.print(f"Adding sibi-flux dependency...")
+        try:
+            from importlib.metadata import version
+            ver = version("sibi-flux")
+            pkg_spec = f"sibi-flux>={ver}"
+        except Exception:
+            pkg_spec = "sibi-flux@latest"
+            
+        subprocess.check_call(["uv", "add", pkg_spec], cwd=project_path)
+
+        # 2b. Add dev dependencies
+        dev_deps = ["black", "notebook", "pytest", "ruff", "httpx", "poethepoet", "PyYAML"]
+        console.print(f"Adding dev dependencies: {', '.join(dev_deps)}...")
+        subprocess.check_call(["uv", "add", "--dev"] + dev_deps, cwd=project_path)
+        
+        # 3. Create Scaffolding Folders (conf, dataobjects, generators)
+        (project_path / "conf").mkdir(exist_ok=True)
+        (project_path / "dataobjects").mkdir(exist_ok=True)
+        (project_path / "generators" / "datacubes").mkdir(parents=True, exist_ok=True)
+        
+        # Add basic .gitkeep
+        (project_path / "conf" / ".gitkeep").touch()
+        (project_path / "dataobjects" / ".gitkeep").touch()
+        (project_path / "generators" / ".gitkeep").touch()
+        # (project_path / "generators" / "datacubes" / ".gitkeep").touch() # Not needed if we write params
+        
+        # 4. Write discovery_params.yaml templates
+        try:
+             from importlib import resources as importlib_resources
+             pass
+        except ImportError:
+             import importlib_resources # type: ignore
+
+        # Access the template resource from sibi_flux.init.templates
+        try:
+            # Modern way
+            ref = importlib_resources.files("sibi_flux.init.templates").joinpath("discovery_params.yaml")
+            template_content = ref.read_text()
+        except AttributeError:
+            # Python < 3.9
+            template_content = importlib_resources.read_text("sibi_flux.init.templates", "discovery_params.yaml")
+
+        # Refactor paths for the new project structure
+        template_content = template_content.replace("solutions/dataobjects/", "dataobjects/")
+        template_content = template_content.replace("solutions.conf", "conf")
+        template_content = template_content.replace(".env.linux", ".env.local")
+
+        (project_path / "generators" / "datacubes" / "discovery_params.yaml").write_text(template_content)
+
+        # Write property_template.yaml
+        try:
+            ref_prop = importlib_resources.files("sibi_flux.init.templates").joinpath("property_template.yaml")
+            prop_content = ref_prop.read_text()
+        except AttributeError:
+            prop_content = importlib_resources.read_text("sibi_flux.init.templates", "property_template.yaml")
+
+        (project_path / "generators" / "datacubes" / "property_template.yaml").write_text(prop_content)
+
+        # Write gen_dc.py
+        try:
+            ref_gen = importlib_resources.files("sibi_flux.init.templates").joinpath("gen_dc.py")
+            gen_content = ref_gen.read_text()
+        except AttributeError:
+            gen_content = importlib_resources.read_text("sibi_flux.init.templates", "gen_dc.py")
+
+        (project_path / "generators" / "datacubes" / "gen_dc.py").write_text(gen_content)
+
+        # 5. Create .env.local
+        (project_path / ".env.local").touch()
+
+        # 6. Create/Update .gitignore
+        gitignore_path = project_path / ".gitignore"
+        gitignore_content = ""
+        if gitignore_path.exists():
+            gitignore_content = gitignore_path.read_text()
+        
+        # Ensure .env is ignored
+        if ".env" not in gitignore_content:
+            with open(gitignore_path, "a") as f:
+                if gitignore_content and not gitignore_content.endswith("\n"):
+                    f.write("\n")
+                f.write("\n# Environment variables\n.env\n.env.local\n.env.*\n")
+        
+        if not gitignore_path.exists():
+             gitignore_path.write_text("# Sibi Flux\n.env\n.env.local\n.env.*\n__pycache__/\n*.pyc\n.DS_Store\n")
+
+        # 7. Configure poe tasks in pyproject.toml
+        pyproject_path = project_path / "pyproject.toml"
+        if pyproject_path.exists():
+            with open(pyproject_path, "a") as f:
+                f.write("\n\n[tool.poe.tasks]\n")
+                f.write('dc-sync = "python generators/datacubes/gen_dc.py sync"\n')
+                f.write('dc-init = "python generators/datacubes/gen_dc.py init"\n')
+                f.write('dc-discover = "python generators/datacubes/gen_dc.py discover"\n')
+                f.write('dc-scan = "python generators/datacubes/gen_dc.py scan"\n')
+                f.write('dc-match = "python generators/datacubes/gen_dc.py match"\n')
+                f.write('dc-map = "python generators/datacubes/gen_dc.py map"\n')
+
+        console.print(f"[bold green]Successfully initialized {project_name}![/bold green]")
+        console.print(f"Created directories: conf/, dataobjects/, generators/datacubes/")
+        console.print(f"Created files: .env.local, .gitignore, generators/datacubes/[discovery_params.yaml, property_template.yaml, gen_dc.py]")
+        console.print(f"cd {project_name}")
+        
+    except subprocess.CalledProcessError as e:
+        console.print(f"[red]Command failed with exit code {e.returncode}[/red]")
+        raise typer.Exit(code=e.returncode)
+    except FileNotFoundError:
+        console.print("[red]Error: 'uv' command not found. Please ensure uv is installed and in your PATH.[/red]")
+        raise typer.Exit(code=1)

sibi_flux/init/discovery_updater.py

@@ -0,0 +1,99 @@
+import yaml
+from pathlib import Path
+from typing import List, Dict, Any
+from sibi_flux.init.env_engine import EnvFile, EnvSection
+from rich.console import Console
+
+console = Console()
+
+class DiscoveryParamsUpdater:
+    """
+    Updates generators/datacubes/discovery_params.yaml to include
+    database connections found in the generated env configuration.
+    """
+
+    @staticmethod
+    def _is_database(section: EnvSection) -> bool:
+        return section.type.upper() in ["POSTGRES", "MYSQL", "CLICKHOUSE", "SQLALCHEMY_DATABASE"]
+
+    @staticmethod
+    def update(project_path: Path, env_data: EnvFile) -> None:
+        params_path = project_path / "generators" / "datacubes" / "discovery_params.yaml"
+        if not params_path.exists():
+            console.print(f"[yellow]Warning: {params_path} not found. Skipping discovery params update.[/yellow]")
+            return
+
+        try:
+            # 1. Read existing YAML
+            with open(params_path, "r") as f:
+                data = yaml.safe_load(f) or {}
+
+            # 2. Extract DBs from Env
+            db_sections = [s for s in env_data.sections if DiscoveryParamsUpdater._is_database(s)]
+            
+            if not db_sections:
+                if "databases" in data:
+                    del data["databases"]
+                    with open(params_path, "w") as f:
+                        yaml.dump(data, f, sort_keys=False, default_flow_style=False)
+                    console.print(f"[yellow]Removed databases section from discovery_params.yaml (no databases found in env).[/yellow]")
+                return
+
+            # Ensure databases list exists
+            if "databases" not in data:
+                data["databases"] = []
+            
+            # Map existing entries by ID to avoid duplicates (upsert behavior)
+            existing_dbs = {db.get("id"): db for db in data["databases"]}
+
+            changes_made = False
+
+            for section in db_sections:
+                db_id = section.name.lower()
+                
+                # Check exclusion? No, default to include all.
+                
+                # Generate entry
+                conf_name = f"{section.name.lower()}_conf"
+                if section.type.upper() == "CLICKHOUSE":
+                    conf_name = "clickhouse_config"
+                
+                # Extract DB Name for db_domain
+                db_domain = db_id # Fallback
+                for var in section.vars:
+                    # Check for common DB name keys. 
+                    # Note: Keys in EnvSection are fully qualified (e.g. REPLICA_DATABASE)
+                    # We check if the suffix matches DATABASE
+                    if var.key.endswith("_DATABASE") or var.key == "DATABASE":
+                         db_domain = var.value
+                         break
+
+                new_entry = {
+                    "id": db_id,
+                    "connection_ref": conf_name,
+                    "db_domain": db_domain,
+                    "import_spec": {
+                        "module": "conf.credentials",
+                        "symbol": conf_name
+                    }
+                }
+                
+                # Upsert
+                if db_id in existing_dbs:
+                    # Update existing? Only if significantly different? 
+                    # For now, let's update strict fields but preserve extra if any.
+                    # Or simpler: Overwrite specific keys.
+                    existing_dbs[db_id].update(new_entry)
+                else:
+                    data["databases"].append(new_entry)
+                    changes_made = True
+
+            # 3. Write back
+            # Use sort_keys=False to preserve order roughly
+            with open(params_path, "w") as f:
+                yaml.dump(data, f, sort_keys=False, default_flow_style=False)
+            
+            console.print(f"[green]Updated discovery_params.yaml with {len(db_sections)} databases.[/green]")
+
+        except Exception as e:
+             console.print(f"[red]Failed to update discovery_params.yaml: {e}[/red]")

sibi_flux/init/env.py

@@ -0,0 +1,86 @@
+from pathlib import Path
+from rich.console import Console
+from typing import Optional
+
+from sibi_flux.init.env_engine import EnvParser
+from sibi_flux.init.env_generator import EnvGenerator
+
+console = Console()
+
+def init_env(project_path: Path, env_file: Optional[Path] = None, cleanup: bool = False, production_mode: bool = False):
+    """
+    Initializes configuration files for the project.
+    Parses .env file (or defaults) and dynamically generates settings.py and credentials.
+    If cleanup is True, removes generated configuration files.
+    If production_mode is True, generates a 'skeleton' settings.py without hardcoded values.
+    """
+    conf_dir = project_path / "conf"
+    
+    if cleanup:
+        settings_path = conf_dir / "settings.py"
+        creds_path = conf_dir / "credentials" / "__init__.py"
+        
+        removed = []
+        if settings_path.exists():
+            settings_path.unlink()
+            removed.append("conf/settings.py")
+        
+        if creds_path.exists():
+            creds_path.unlink()
+            removed.append("conf/credentials/__init__.py")
+            
+        if removed:
+            console.print(f"[green]Removed: {', '.join(removed)}[/green]")
+        else:
+            console.print("[yellow]No configuration files found to remove.[/yellow]")
+        return
+
+    # Parse Environment
+    if env_file:
+         if not env_file.exists():
+             console.print(f"[red]Error: Environment file {env_file} does not exist.[/red]")
+             return
+         
+         content = env_file.read_text().strip()
+         if not content or all(line.strip().startswith('#') for line in content.splitlines() if line.strip()):
+             console.print(f"[yellow]Environment file {env_file} is empty or contains only comments. No action taken.[/yellow]")
+             return
+             
+         env_data = EnvParser.parse(content)
+         env_filename = env_file.name
+    else:
+         # Implicit default: try to find .env, but if not found or empty, maybe generate skeleton?
+         # User request implies: "if an .env file is empty, message that nothing will be done"
+         # But if NO file passed, we assume .env in project root.
+         
+         default_env = project_path / ".env"
+         if not default_env.exists():
+              console.print(f"[yellow]No .env file found at {default_env}. Skipping generation.[/yellow]")
+              return
+
+         content = default_env.read_text().strip()
+         if not content or all(line.strip().startswith('#') for line in content.splitlines() if line.strip()):
+             console.print(f"[yellow]Environment file .env is empty or contains only comments. No action taken.[/yellow]")
+             return
+             
+         env_data = EnvParser.parse(content)
+         env_filename = ".env"
+
+    conf_dir.mkdir(exist_ok=True)
+    
+    # 1. settings.py
+    settings_code = EnvGenerator.generate_pydantic_code(env_data, env_file_name=env_filename, production_mode=production_mode)
+    (conf_dir / "settings.py").write_text(settings_code)
+    console.print(f"[green]Created conf/settings.py (parsed {len(env_data.sections)} sections)[/green]")
+
+    # 2. credentials/__init__.py
+    creds_dir = conf_dir / "credentials"
+    creds_dir.mkdir(exist_ok=True)
+    
+    creds_code = EnvGenerator.generate_credentials_code(env_data)
+    (creds_dir / "__init__.py").write_text(creds_code)
+    console.print(f"[green]Created conf/credentials/__init__.py[/green]")
+
+    # 3. Update discovery_params.yaml
+    from sibi_flux.init.discovery_updater import DiscoveryParamsUpdater
+    DiscoveryParamsUpdater.update(project_path, env_data)

sibi_flux/init/env_engine.py

@@ -0,0 +1,151 @@
+from pydantic import BaseModel, Field
+from typing import List, Dict, Optional, Any
+import re
+from pathlib import Path
+from collections import defaultdict
+
+class EnvVar(BaseModel):
+    key: str
+    value: str
+    comment: Optional[str] = None
+
+class EnvSection(BaseModel):
+    name: str # e.g. "MAIN_DB"
+    type: str = "GENERIC" # e.g. "POSTGRES", "S3"
+    vars: List[EnvVar]
+    resource_id: Optional[str] = None
+
+class EnvFile(BaseModel):
+    sections: List[EnvSection]
+    orphan_vars: List[EnvVar] = Field(default_factory=list)
+
+class EnvParser:
+    SECTION_REGEX = re.compile(r"^#\s*\[(.*)\]\s*(?:\((.*)\))?") # Matches # [NAME] or # [NAME](TYPE)
+    
+    @staticmethod
+    def parse(content: str) -> EnvFile:
+        sections = []
+        orphans = []
+        current_section = None
+        
+        for line in content.splitlines():
+            line = line.strip()
+            if not line: continue
+            
+            # Section Header detection
+            sec_match = EnvParser.SECTION_REGEX.match(line)
+            if sec_match:
+                if current_section: sections.append(current_section)
+                name = sec_match.group(1)
+                type_hint = sec_match.group(2) if sec_match.lastindex >= 2 else "GENERIC"
+                current_section = EnvSection(name=name, type=type_hint or "GENERIC", vars=[])
+                continue
+                
+            # Var detection
+            if "=" in line and not line.startswith("#"):
+                key, val = line.split("=", 1)
+                # Parse inline comments
+                comment = None
+                if "#" in val:
+                    val, comment = val.split("#", 1)
+                    comment = comment.strip()
+                
+                var = EnvVar(key=key.strip(), value=val.strip(), comment=comment)
+                
+                if current_section:
+                    current_section.vars.append(var)
+                else:
+                    orphans.append(var)
+        
+        if current_section: sections.append(current_section)
+        
+        env_file = EnvFile(sections=sections, orphan_vars=orphans)
+        
+        return EnvParser.heuristic_grouping(env_file)
+
+    @staticmethod
+    def heuristic_grouping(env_file: EnvFile) -> EnvFile:
+        """
+        Groups orphan variables into sections based on common prefixes.
+        """
+        if not env_file.orphan_vars:
+            return env_file
+            
+        groups = defaultdict(list)
+        others = []
+        
+        for var in env_file.orphan_vars:
+            if "_" in var.key:
+                prefix = var.key.split("_", 1)[0]
+                groups[prefix].append(var)
+            else:
+                others.append(var)
+                
+        new_sections = list(env_file.sections)
+        
+        for prefix, vars_list in groups.items():
+            # Check if section already exists
+            existing = next((s for s in new_sections if s.name == prefix), None)
+            if existing:
+                existing.vars.extend(vars_list)
+            else:
+                # Type Inference
+                inferred_type = "GENERIC"
+                upper_prefix = prefix.upper()
+                
+                # 1. Name-based inference
+                KNOWN_TYPES = ["POSTGRES", "MYSQL", "CLICKHOUSE", "REDIS", "S3", "AIRFLOW", "OSMNX", "SENTINEL", "OPENOBSERVE", "MCP", "DASK"]
+                if upper_prefix in KNOWN_TYPES:
+                    inferred_type = upper_prefix
+                
+                # 2. Key-based inference (if still GENERIC)
+                if inferred_type == "GENERIC":
+                    keys = []
+                    for v in vars_list:
+                        k = v.key.upper()
+                        if k.startswith(upper_prefix + "_"):
+                            keys.append(k[len(upper_prefix)+1:])
+                        else:
+                            keys.append(k)
+                    
+                    keys_set = set(keys)
+                    
+                    # Signatures
+                    if any("AWS_ACCESS_KEY" in k or "S3_" in k or "FS_KEY" in k for k in [v.key.upper() for v in vars_list]): 
+                        inferred_type = "S3"
+                    elif "HOST" in keys_set and "PORT" in keys_set:
+                        if "DB_INDEX" in keys_set or "REDIS_DB" in keys_set:
+                            inferred_type = "REDIS"
+                        elif "DATABASE" in keys_set or "DBNAME" in keys_set or "USER" in keys_set:
+                            inferred_type = "POSTGRES" 
+                        else:
+                            pass
+                    elif "API_URL" in keys_set and ("USERNAME" in keys_set or "PASSWORD" in keys_set):
+                         inferred_type = "AIRFLOW" 
+                
+                new_sections.append(EnvSection(name=prefix, type=inferred_type, vars=vars_list))
+                
+        return EnvFile(sections=new_sections, orphan_vars=others)
+
+    @staticmethod
+    def render(env_file: EnvFile) -> str:
+        lines = []
+        
+        # Orphans first
+        for var in env_file.orphan_vars:
+            line = f"{var.key}={var.value}"
+            if var.comment: line += f" # {var.comment}"
+            lines.append(line)
+            
+        if lines: lines.append("")
+            
+        # Sections
+        for section in env_file.sections:
+            lines.append(f"# [{section.name}]({section.type})")
+            for var in section.vars:
+                line = f"{var.key}={var.value}"
+                if var.comment: line += f" # {var.comment}"
+                lines.append(line)
+            lines.append("")
+            
+        return "\n".join(lines)