sibi-flux 2025.12.0__tar.gz → 2026.1.2__tar.gz

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: sibi-flux
-Version: 2025.12.0
+Version: 2026.1.2
 Summary: Sibi Toolkit: A collection of tools for Data Analysis/Engineering.
 Author: Luis Valverde
 Author-email: Luis Valverde <lvalverdeb@gmail.com>
@@ -27,6 +27,8 @@ Requires-Dist: httpx>=0.28.1
 Requires-Dist: opentelemetry-api>=1.38.0
 Requires-Dist: opentelemetry-exporter-otlp>=1.38.0
 Requires-Dist: opentelemetry-sdk>=1.38.0
+Requires-Dist: deep-translator>=1.11.4
+Requires-Dist: pyyaml>=6.0.3
 Requires-Dist: sibi-flux[distributed,geospatial,mcp] ; extra == 'complete'
 Requires-Dist: distributed>=2025.11.0 ; extra == 'distributed'
 Requires-Dist: osmnx>=2.0.7 ; extra == 'geospatial'
@@ -281,3 +283,43 @@ async with GenericMcpClient(url="http://localhost:8000/sse") as client:
     # Call a tool
     vat = await client.call_tool("calculate_vat", arguments={"amount": 100.0})
 ```
+
+### 8. Datacube Generation (`gen_dc.py`)
+
+Automate the creation of Datacube classes and Field Maps from your database schema.
+
+**Configuration (`discovery_params.yaml`)**
+
+Define your generation rules in a hierarchical configuration file:
+
+```yaml
+defaults:
+  backend: sqlalchemy
+  class_suffix: Dc
+
+discovery:
+  whitelist_file: whitelist.yaml
+  rules_file: discovery_rules.yaml
+
+generation:
+  enable_field_maps: true
+```
+
+**Commands**
+
+1.  **Discover**: Introspects the database and updates your whitelist/registry.
+    ```bash
+    uv run poe dc-discover
+    ```
+    *   **Whitelist**: Explicitly define tables to generate. Support `custom_name` to override class names.
+    *   **Rules**: Regex-based patterns to match tables.
+
+2.  **Sync**: Generates Python code (Datacubes and Field Maps) based on the registry.
+    ```bash
+    uv run poe dc-sync --force
+    ```
+
+**Key Features**
+- **Custom Naming**: Add `custom_name: MyCube` to `whitelist.yaml` to override generated names.
+- **Hierarchical Config**: Strict validation of generation parameters.
+- **Field Maps**: Auto-generates type-safe mapping files for every table.

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/README.md

@@ -231,3 +231,43 @@ async with GenericMcpClient(url="http://localhost:8000/sse") as client:
     # Call a tool
     vat = await client.call_tool("calculate_vat", arguments={"amount": 100.0})
 ```
+
+### 8. Datacube Generation (`gen_dc.py`)
+
+Automate the creation of Datacube classes and Field Maps from your database schema.
+
+**Configuration (`discovery_params.yaml`)**
+
+Define your generation rules in a hierarchical configuration file:
+
+```yaml
+defaults:
+  backend: sqlalchemy
+  class_suffix: Dc
+
+discovery:
+  whitelist_file: whitelist.yaml
+  rules_file: discovery_rules.yaml
+
+generation:
+  enable_field_maps: true
+```
+
+**Commands**
+
+1.  **Discover**: Introspects the database and updates your whitelist/registry.
+    ```bash
+    uv run poe dc-discover
+    ```
+    *   **Whitelist**: Explicitly define tables to generate. Support `custom_name` to override class names.
+    *   **Rules**: Regex-based patterns to match tables.
+
+2.  **Sync**: Generates Python code (Datacubes and Field Maps) based on the registry.
+    ```bash
+    uv run poe dc-sync --force
+    ```
+
+**Key Features**
+- **Custom Naming**: Add `custom_name: MyCube` to `whitelist.yaml` to override generated names.
+- **Hierarchical Config**: Strict validation of generation parameters.
+- **Field Maps**: Auto-generates type-safe mapping files for every table.

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "sibi-flux"
-version = "2025.12.0"
+version = "2026.1.2"
 description = "Sibi Toolkit: A collection of tools for Data Analysis/Engineering."
 readme = "README.md"
 authors = [
@@ -33,6 +33,8 @@ dependencies = [
     "opentelemetry-api>=1.38.0",
     "opentelemetry-exporter-otlp>=1.38.0",
     "opentelemetry-sdk>=1.38.0",
+    "deep-translator>=1.11.4",
+    "pyyaml>=6.0.3",
 ]
 
 [project.optional-dependencies]
@@ -59,6 +61,9 @@ complete = [
     "sibi-flux[distributed,geospatial,mcp]"
 ]
 
+[project.scripts]
+sibi-flux = "sibi_flux.cli:app"
+
 
 [dependency-groups]
 dev = [
@@ -89,6 +94,35 @@ build-backend = "uv_build"
 module-root = "src"
 module-name = ["sibi_flux", "sibi_dst"]
 
+[tool.uv.workspace]
+members = [
+    "test-project-alpha",
+    "test-project-beta",
+    "test-project-gamma",
+    "test-project-delta",
+    "test-project-epsilon",
+    "test-project-zeta",
+    "test-project-eta",
+    "test-project-theta",
+    "test-project-iota",
+    "test-project-kappa",
+    "test-project-lambda",
+    "test-project-alpha/latest",
+    "test-project-mu",
+    "test-project-mu/latest",
+    "test-project-xi",
+    "test-project-xi/latest",
+    "test-project-omicron",
+    "test-project-omicron/latest",
+    "test-project-pi",
+    "test-project-pi/latest",
+    "test-project-rho",
+    "test-project-rho/latest",
+    "test-project-sigma",
+    "test-project-sigma/latest",
+    "test-project-tau",
+]
+
 [tool.pytest.ini_options]
 pythonpath = ["src", "."]
 testpaths = ["tests"]
@@ -97,6 +131,7 @@ filterwarnings = ["ignore::DeprecationWarning"]
 
 
 [tool.poe.tasks]
+publish = { cmd = "uv publish", envfile = ".env" }
 dev = """
 uvicorn solutions.main:app
     --reload
@@ -108,6 +143,17 @@ uvicorn solutions.main:app
 """
 test = { cmd = "pytest tests/"}
 lint = "black src/"
+build = "uv build"
+dc-sync = "python solutions/generators/datacubes/gen_dc.py sync"
+dc-init = "python solutions/generators/datacubes/gen_dc.py init"
+dc-discover = "python solutions/generators/datacubes/gen_dc.py discover"
+dc-scan = "python solutions/generators/datacubes/gen_dc.py scan"
+dc-match = "python solutions/generators/datacubes/gen_dc.py match"
+dc-map = "python solutions/generators/datacubes/gen_dc.py map"
+
+[tool.poe.tasks.release]
+sequence = ["build","publish"]
+envfile = ".env"  # Loads the token for the whole sequence
 
 [tool.commitizen]
 name = "cz_conventional_commits"

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/src/sibi_flux/__init__.py

@@ -21,11 +21,12 @@ from sibi_flux.df_validator._df_validator import DfValidator
 
 # Artifacts
 from sibi_flux.artifacts import ParquetArtifact, BaseArtifact as Artifact
-from sibi_flux.readers.parquet import ParquetReader
+from sibi_flux.parquet import ParquetReader
 
 # Utilities (Sub-packages)
 from sibi_flux import dask_cluster
-from sibi_flux.utils import boilerplate, parquet_saver, clickhouse_writer
+from sibi_flux.utils import clickhouse_writer
+from sibi_flux import parquet
 
 
 __all__ = [
@@ -43,7 +44,6 @@ __all__ = [
     "Artifact",
     "ParquetReader",
     "dask_cluster",
-    "boilerplate",
-    "parquet_saver",
+    "parquet",
     "clickhouse_writer",
 ]

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/src/sibi_flux/artifacts/parquet_engine/executor.py

@@ -13,7 +13,7 @@ import functools
 from tqdm import tqdm
 import pandas as pd
 from sibi_flux.core import ManagedResource
-from sibi_flux.utils.parquet_saver import ParquetSaver
+from sibi_flux.parquet import ParquetSaver
 
 from sibi_flux.utils import ensure_slash
 from sibi_flux.utils.retry import with_retry

sibi_flux-2026.1.2/src/sibi_flux/cli.py

@@ -0,0 +1,45 @@
+import typer
+from typing import Optional
+from pathlib import Path
+from rich.console import Console
+from sibi_flux.init.core import initialize_project
+
+app = typer.Typer(help="Sibi Flux CLI")
+console = Console()
+
+@app.callback()
+def callback():
+    """
+    Sibi Flux CLI
+    """
+
+@app.command()
+def init(
+    project_name: str = typer.Argument(..., help="Name of the project to create"),
+    lib: bool = typer.Option(False, "--lib", help="Initialize as a library project (passed to uv init)"),
+    app: bool = typer.Option(False, "--app", help="Initialize as an application project (passed to uv init)")
+):
+    """
+    Initialize a new Sibi Flux project.
+    
+    Creates a new directory <project_name>, initializes it with 'uv',
+    and adds 'sibi-flux' as a dependency.
+    """
+    initialize_project(project_name, lib, app)
+
+@app.command()
+def env(
+    project_path: Path = typer.Argument(Path("."), help="Project root directory"),
+    env_file: Optional[Path] = typer.Option(None, "--env-file", "-e", help="Path to environment file (defaults to .env)"),
+    cleanup: bool = typer.Option(False, "--cleanup", help="Remove existing configuration files"),
+    production: bool = typer.Option(False, "--production", "-p", help="Generate production skeleton (no hardcoded values)"),
+):
+    """
+    Initialize configuration files (settings.py, credentials) based on .env
+    """
+    from sibi_flux.init.env import init_env
+    init_env(project_path, env_file, cleanup=cleanup, production_mode=production)
+
+
+if __name__ == "__main__":
+    app()

sibi_flux-2026.1.2/src/sibi_flux/config/__init__.py

@@ -0,0 +1,3 @@
+from .manager import ConfigManager
+
+__all__ = ["ConfigManager"]

{sibi_flux-2025.12.0/src/sibi_flux/conf → sibi_flux-2026.1.2/src/sibi_flux/config}/settings.py

@@ -1,4 +1,4 @@
-from typing import Optional, Any
+from typing import Optional, Any, ClassVar
 from pydantic import SecretStr
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
@@ -9,6 +9,8 @@ class SibiBaseSettings(BaseSettings):
     model_config = SettingsConfigDict(
         env_file=".env", env_file_encoding="utf-8", extra="ignore"
     )
+    
+    conf_name: ClassVar[str] = ""
 
 
 class FsSettings(SibiBaseSettings):
@@ -84,13 +86,8 @@ class DatabaseSettings(SibiBaseSettings):
     """Generic SQL Database settings."""
 
     db_url: str = "sqlite:///:memory:"
-
-
-class ClickhouseBaseSettings(SibiBaseSettings):
-    """Base settings for ClickHouse connection."""
-
     host: str = "localhost"
-    port: int = 8123
+    port: int = 5432
     database: str = "default"
     user: str = "default"
     password: SecretStr = SecretStr("secret")
@@ -102,9 +99,12 @@ class ClickhouseBaseSettings(SibiBaseSettings):
             "dbname": self.database,
             "user": self.user,
             "password": self.password.get_secret_value() if self.password else None,
+            "db_url": self.db_url,
         }
 
 
+
+
 class RedisBaseSettings(SibiBaseSettings):
     """Base settings for Redis connection."""
 

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/src/sibi_flux/dask_cluster/async_core.py

@@ -4,6 +4,7 @@ Async utilities for Dask operations.
 
 from typing import Any, Optional
 import asyncio
+
 try:
     from dask.distributed import Client
 except ImportError:

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/src/sibi_flux/dask_cluster/client_manager.py

@@ -23,6 +23,7 @@ import shutil
 
 try:
     from dask.distributed import Client, LocalCluster, get_client
+
     HAS_DISTRIBUTED = True
 except ImportError:
     Client = object
@@ -231,8 +232,10 @@ class DaskClientMixin:
     def _init_dask_client(self, **kwargs) -> None:
         self._init_params = kwargs
         if not HAS_DISTRIBUTED:
-             self.logger.info("Dask Distributed not installed. Skipping cluster initialization.")
-             return
+            self.logger.info(
+                "Dask Distributed not installed. Skipping cluster initialization."
+            )
+            return
 
         if kwargs.get("dask_client"):
             self.dask_client = kwargs["dask_client"]

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/src/sibi_flux/dask_cluster/core.py

@@ -14,15 +14,18 @@ from typing import Any, Callable, Dict, List, Optional, TypeVar
 import dask
 import dask.dataframe as dd
 import pandas as pd
+
 try:
     from dask.distributed import Client, Future
     from dask.distributed import wait as dask_wait
 except ImportError:
     Client = object
     Future = object
+
     def dask_wait(*args, **kwargs):
         pass
 
+
 # Project-specific imports
 from .client_manager import get_persistent_client
 from .exceptions import RECOVERABLE_COMMS

{sibi_flux-2025.12.0 → sibi_flux-2026.1.2}/src/sibi_flux/datacube/_data_cube.py

@@ -175,9 +175,10 @@ class Datacube(DfHelper):
             # but we log it for debugging.
             self.logger.debug(f"Schema inference skipped: {e}")
 
-    def _validate(self, df: DataFrameType) -> DataFrameType:
+    def validate_data(self, df: DataFrameType) -> DataFrameType:
         """
         Runs DfValidator if a schema is configured.
+        Overrides BaseDatacube hook.
         """
         schema = self.config.get("validation_schema")
         if not schema:
@@ -203,6 +204,14 @@ class Datacube(DfHelper):
 
         return validator.get_df()
 
+    async def avalidate_data(self, df: DataFrameType) -> DataFrameType:
+        """
+        Asynchronous validation hook.
+        Offloads synchronous validation (CPU bound) to a thread.
+        """
+        import asyncio
+        return await asyncio.to_thread(self.validate_data, df)
+
     def get_ddl(self, table_name: Optional[str] = None) -> str:
         """
         Generates ClickHouse DDL for the current cube.
@@ -230,7 +239,7 @@ class Datacube(DfHelper):
             # 3. Apply Transform Hook
             df = self.fix_data(df, **kwargs)
             # 4. Validate
-            df = self._validate(df)
+            df = self.validate_data(df)
         else:
             self.logger.debug(f"No data loaded by {self.__class__.__name__}")
 
@@ -254,7 +263,7 @@ class Datacube(DfHelper):
             # 3. Apply Async Transform Hook
             df = await self.afix_data(df, **kwargs)
             # 4. Validate (CPU bound)
-            df = await asyncio.to_thread(self._validate, df)
+            df = await self.avalidate_data(df)
         else:
             self.logger.debug(f"No data loaded by {self.__class__.__name__}")