tdfs4ds 0.2.4.45__py3-none-any.whl → 0.2.4.47__py3-none-any.whl

tdfs4ds/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.2.4.45'
+__version__ = '0.2.4.47'
 import logging
 import json
 
@@ -24,8 +24,8 @@ from tdfs4ds.feature_store.feature_query_retrieval import get_available_entity_i
 from tdfs4ds.process_store.process_followup import follow_up_report
 from tdfs4ds.dataset.dataset_catalog import DatasetCatalog, Dataset
 
-DATA_DOMAIN             = None
-SCHEMA                  = None
+DATA_DOMAIN                  = None
+SCHEMA                       = None
 FEATURE_CATALOG_NAME         = 'FS_FEATURE_CATALOG'
 FEATURE_CATALOG_NAME_VIEW    = 'FS_V_FEATURE_CATALOG'
 PROCESS_CATALOG_NAME         = 'FS_PROCESS_CATALOG'

tdfs4ds/feature_store/feature_data_processing.py

@@ -286,12 +286,31 @@ def prepare_feature_ingestion(df, entity_id, feature_names, feature_versions=Non
 
     # Execute: create volatile table and test unicity
     try:
-        tdml.DataFrame.from_query(nested_query).to_sql(
-            table_name    = volatile_table_name,
-            temporary     = True,
-            primary_index = primary_index.split(','),
-            if_exists     = 'replace'
-        )
+        try:
+            tdml.execute_sql(f"DROP TABLE {_get_database_username()}.{volatile_table_name};")
+        except:
+            pass  # ignore if not exists
+        query_volatile = f"""
+            CREATE VOLATILE TABLE {volatile_table_name} AS
+            (
+            WITH MY_PROCESSING AS (
+            SELECT * FROM {df._table_name}
+            )
+            SELECT 
+            {var_temp3},
+            {var_temp2}
+            FROM MY_PROCESSING
+            ) WITH DATA
+            PRIMARY INDEX ({primary_index})
+            ON COMMIT PRESERVE ROWS
+        """
+        execute_query(query_volatile)
+        #tdml.DataFrame.from_query(nested_query).to_sql(
+        #    table_name    = volatile_table_name,
+        #    temporary     = True,
+        #    primary_index = primary_index.split(','),
+        #    if_exists     = 'replace'
+        #)
         nb_duplicates = tdml.execute_sql(query_test_unicity).fetchall()[0][0]
         if nb_duplicates is not None and nb_duplicates > 0:
             logger_safe("error", "The process generates %s duplicates", nb_duplicates)
@@ -991,6 +1010,8 @@ def prepare_feature_ingestion_tdstone2(df, entity_id):
     PRIMARY INDEX ({primary_index})
     {volatile_expression}
     """
+
+    
     # Execute the SQL query to create the volatile table.
     try:
         tdml.execute_sql(query)

tdfs4ds/feature_store/feature_store_management.py

@@ -217,7 +217,7 @@ def feature_store_table_creation(entity_id, feature_type, if_exists='fail', prim
     query2 = f"CREATE INDEX (FEATURE_ID) ON {tdfs4ds.SCHEMA}.{table_name};"
 
     # SQL query to comment the table indicating the entity id it is related to
-    query3 = f"COMMENT ON TABLE {tdfs4ds.SCHEMA}.{table_name} IS ' entity id {ENTITY_ID}'"
+    query3 = f"COMMENT ON TABLE {tdfs4ds.SCHEMA}.{table_name} IS 'Feature Store Table'"
 
     # SQL query to create the view
     query_view = f"""

tdfs4ds/genai/__init__.py

@@ -0,0 +1,351 @@
+from typing import Sequence, List
+
+from langchain_openai import ChatOpenAI
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.output_parsers import JsonOutputParser
+from langchain_core.runnables import Runnable, RunnableLambda
+from langchain_core.messages import AIMessage
+
+
+def build_llm(
+    llm_service: str = "https://api-dmproject.myddns.me/v1",
+    api_key: str = "YOUR_API_KEY_HERE",  # 🔒 Better not ship the real key in code
+    model_id: str = "mistralai/Ministral-8B-Instruct-2410",
+    temperature: float = 0.2,
+    timeout: int = 120,
+) -> ChatOpenAI:
+    """
+    Build and return a ChatOpenAI client pointed at your vLLM/OpenAI-compatible endpoint.
+    """
+    return ChatOpenAI(
+        base_url=llm_service,
+        api_key=api_key,
+        model=model_id,
+        temperature=temperature,
+        timeout=timeout,
+    )
+
+
+from typing import Sequence
+
+def build_documentation_json_schema(columns, provider="generic"):
+    """
+    Build a provider-appropriate JSON Schema used to enforce strict JSON output
+    for SQL column documentation across multiple LLM backends.
+
+    This function returns different schema shapes depending on the LLM provider,
+    because each ecosystem uses a different structured-output mechanism:
+
+    Provider Modes
+    --------------
+    - provider="openai", "azure"
+        Returns the JSON Schema wrapped in OpenAI's `response_format={"type": "json_schema", ...}`
+        structure. Supported by GPT-4.1, GPT-4o, GPT-3.5-Turbo, and Azure OpenAI.
+
+    - provider="anthropic", "claude"
+        Returns an Anthropic *tool schema* definition. Claude 3.x models use tool
+        schemas to enforce strict JSON output.
+
+    - provider="ollama"
+        Returns the raw JSON schema that Ollama expects under the `format=` parameter
+        of the generate API. (Ollama 0.2+ supports response schemas.)
+
+    - provider="vllm"
+        Returns plain JSON Schema for use with vLLM's `guided_json` constrained decoding.
+
+    - provider="bedrock"
+        Bedrock Claude follows the Anthropic tool schema format.
+        Bedrock Llama / Titan accept plain JSON schema. This function returns the base
+        schema and leaves the final wrapping to the caller.
+
+    - provider="generic"
+        Returns plain JSON schema. Useful for LLM backends that do not support
+        constrained decoding, prompt-only JSON generation, or post-processing repair.
+
+    Parameters
+    ----------
+    columns : list[str]
+        Column names to include as required JSON object keys. Each column will map
+        to a string description generated by the model.
+
+    provider : str, optional
+        The model provider or backend type. Determines the structural format
+        required for constrained generation. One of:
+        "openai", "anthropic", "ollama", "vllm", "bedrock", "generic".
+
+    Returns
+    -------
+    dict
+        A dictionary representing the JSON Schema or provider-specific wrapper
+        used to enforce strict JSON output during LLM generation.
+
+    Notes
+    -----
+    - All schemas require that:
+        * the output be a JSON object
+        * keys match exactly the column names
+        * all values be strings
+        * additional properties be disallowed
+
+    - Not all providers enforce schemas equally:
+        * OpenAI, Claude, and vLLM offer hard guarantees.
+        * Ollama enforces schema reasonably well.
+        * Generic models may require post-processing.
+    """
+    # Base JSON schema — used directly by vLLM, Ollama, Bedrock, fallback
+    base_schema = {
+        "type": "object",
+        "properties": {col: {"type": "string"} for col in columns},
+        "required": list(columns),
+        "additionalProperties": False,
+    }
+
+    # --- Provider-specific formats ---
+
+    if provider.lower() in ("openai", "azure", "azure-openai"):
+        # OpenAI's required wrapper structure
+        return {
+            "type": "json_schema",
+            "json_schema": {
+                "name": "ColumnDocumentation",
+                "schema": base_schema,
+                "strict": True,
+            }
+        }
+
+    if provider.lower() in ("anthropic", "claude"):
+        # Anthropic tool schema
+        # You embed this inside the "tools" field when calling the model
+        return {
+            "name": "column_documentation",
+            "description": "Generate documentation for SQL output columns.",
+            "input_schema": base_schema
+        }
+
+    if provider.lower() == "ollama":
+        # Ollama's output format schema (unwrapped JSON schema)
+        # Returned directly in: generate(..., format=schema)
+        return base_schema
+
+    if provider.lower() in ("vllm", "openai-compatible"):
+        # vLLM's guided_json uses *plain JSON Schema*
+        # so return base_schema exactly
+        return base_schema
+
+    if provider.lower() == "bedrock":
+        # Bedrock Claude uses Anthropic schema
+        # Bedrock Llama uses plain JSON schema
+        # Return base_schema and let caller choose
+        return base_schema
+
+    # Fallback: generic JSON schema
+    return base_schema
+
+
+def build_sql_documentation_chain(llm: ChatOpenAI, columns: Sequence[str], provider: str="vllm", json_constraint=True) -> Runnable:
+    """
+    Build a LangChain Runnable that generates business-focused documentation
+    for a list of SQL output columns, with optional provider-specific JSON
+    constraints (vLLM, OpenAI, Ollama, etc.).
+
+    The resulting chain expects two input variables:
+        - sql_query: str         → the SQL query whose output is being documented
+        - columns_str: str       → formatted list of columns (e.g. "- col1\n- col2")
+
+    Parameters
+    ----------
+    llm : ChatOpenAI
+        The language model interface (may point to vLLM, OpenAI, Ollama, etc.).
+    columns : Sequence[str]
+        List of columns that must appear as keys in the output JSON.
+    provider : str, optional (default="vllm")
+        Indicates which structured-output mechanism to use.
+        Supported values:
+            - "vllm"               → uses `guided_json` for strict JSON output
+            - "openai" / "azure"   → uses OpenAI JSON Schema via `response_format`
+            - "ollama"             → uses Ollama's `format=` schema
+            - "openai-compatible"  → alias for vLLM-style guided decoding
+            - any other value      → fall back to unconstrained text output
+    json_constraint : bool, optional (default=True)
+        If True:
+            - a JSON Schema is generated from the column list
+            - provider-specific constrained decoding is applied
+        If False:
+            - the chain does not enforce JSON structure at the LLM level
+            - the model is only guided by the prompt (weaker guarantees)
+
+    Returns
+    -------
+    Runnable
+        A LangChain Runnable that executes:
+            prompt → LLM (optionally schema-guided) → JSON parser
+
+        When invoked with:
+            {
+                "sql_query": "...",
+                "columns_str": "- column1\n- column2\n..."
+            }
+
+        It returns:
+            dict[str, str]
+                A mapping of each requested column name to a short,
+                business-oriented description (≤ 5 sentences).
+
+    Notes
+    -----
+    - The chain enforces valid JSON when possible:
+        * vLLM → `guided_json`
+        * OpenAI → `response_format={"type": "json_schema", ...}`
+        * Ollama → `format=<schema>`
+    - For unsupported providers, the model may emit imperfect JSON.
+    - Descriptions focus on business meaning, business logic,
+      and optionally technical details only when relevant.
+    """
+    prompt = ChatPromptTemplate.from_template(
+        """
+You are a data documentation assistant.
+
+Your target audience is business users.
+Your explanations must focus primarily on the business meaning and business logic of each column,
+and you may add technical details only when they meaningfully clarify the business context.
+
+Given:
+1. A SQL query.
+2. A list of output columns that must be documented.
+
+Your job:
+- For each column in the provided list, write a clear and concise explanation of what the column represents from a business perspective.
+- Describe the business logic behind how the value is derived or used within the context of the SQL query.
+- Add technical details only if relevant and only to help a business audience understand the concept.
+- Each description must be at most 5 sentences.
+- Do not include any columns that are not in the provided list.
+- If a column name is ambiguous, infer its meaning from the SQL query as best as possible and say so.
+- If you cannot infer anything meaningful, state that clearly (still within 3 sentences).
+- Answer in {language}
+Output format (very important):
+- Return ONLY a valid JSON object.
+- Each top-level key must be exactly the column name.
+- Each value must be a single string with the description.
+
+Example of the required format:
+{{
+  "customer_id": "Unique identifier of the customer placing the order, used to track customers at a business level. Also serves as the technical key linking orders to customer records.",
+  "order_date": "The business date when the order was created. It represents the transaction date used for reporting and may reflect the source system’s timestamp."
+}}
+
+Now generate documentation.
+
+SQL query:
+```sql
+{sql_query}
+```
+Columns to document (only document these):
+{columns_str}
+"""
+    )
+    parser = JsonOutputParser()
+    # ✅ Final chain: prompt -> grammar-constrained LLM -> JSON parser
+    if not json_constraint:
+        chain: Runnable = prompt | llm | parser
+        return chain
+
+    schema = build_documentation_json_schema(columns, provider=provider)
+    # ---- provider-specific LLM call wrapper ----
+    provider_l = provider.lower()
+
+    if provider_l in ("vllm", "openai-compatible"):
+        # vLLM guided_json
+        def _call_llm(messages):
+            return llm.invoke(
+                messages,
+                extra_body={"guided_json": schema},
+            )
+
+    elif provider_l in ("openai", "azure", "azure-openai"):
+        # OpenAI / Azure OpenAI JSON schema via response_format
+        def _call_llm(messages):
+            return llm.invoke(
+                messages,
+                response_format=schema,
+            )
+
+    elif provider_l == "ollama":
+        # Ollama's `format` parameter (schema directly)
+        def _call_llm(messages):
+            return llm.invoke(
+                messages,
+                format=schema,
+            )
+
+    else:
+        # Fallback: no hard structure, rely on prompt & parser
+        def _call_llm(messages):
+            return llm.invoke(messages)
+
+    constrained_llm = RunnableLambda(_call_llm)
+    # Final chain: prompt -> LLM (schema-guided) -> JSON parser
+    def _parse(ai_msg: AIMessage):
+        raw = ai_msg.content
+        return parser.parse(raw)
+
+    chain: Runnable = prompt | constrained_llm | RunnableLambda(_parse)
+    return chain
+
+def run_sql_documentation(chain: Runnable, sql_query: str, columns_to_document: Sequence[str], language: str = "English"):
+    """
+    Execute a previously constructed SQL-documentation chain and return
+    business-friendly documentation for the specified SQL output columns.
+
+    This function prepares the chain inputs (SQL query, formatted column list,
+    target language) and invokes the chain. The chain itself must have been
+    created using `build_sql_documentation_chain()`, which ensures the model
+    produces structured JSON suitable for parsing.
+
+    Parameters
+    ----------
+    chain : Runnable
+        A LangChain Runnable returned by `build_sql_documentation_chain()`.
+        This Runnable encapsulates:
+            - the prompt template
+            - a provider-specific LLM invocation (with or without JSON constraints)
+            - a JSON output parser
+
+    sql_query : str
+        The SQL query whose resulting columns should be documented. This query is
+        shown to the model so it can infer business logic, derivation rules, and
+        column meaning.
+
+    columns_to_document : Sequence[str]
+        The list of column names that must appear as keys in the output JSON.
+        Only these columns will be documented. The order does not matter.
+
+    language : str, optional (default="English")
+        The target output language for the generated documentation.
+        This value is passed into the prompt’s `{language}` variable.
+        Examples: "English", "French", "German", "Spanish", "Japanese".
+
+    Returns
+    -------
+    dict[str, str]
+        A dictionary mapping each column name to a human-readable, business-oriented
+        description generated by the model. Example:
+            {
+                "customer_id": "Unique customer identifier used for ...",
+                "order_date": "Business date when the order was created ..."
+            }
+
+    Notes
+    -----
+    - The output format is determined by the chain's JSON parser. If the model
+      fails to produce valid JSON (e.g., due to unsupported constraints),
+      a `OutputParserException` may be raised.
+    - The resulting descriptions are typically ≤ 5 sentences per column, unless
+      modified in the chain's prompt.
+    """
+    columns_str = "\n".join(f"- {col}" for col in columns_to_document)
+
+    return chain.invoke({
+        "sql_query": sql_query,
+        "columns_str": columns_str,
+        "language" : language
+    })

{tdfs4ds-0.2.4.45.dist-info → tdfs4ds-0.2.4.47.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tdfs4ds
-Version: 0.2.4.45
+Version: 0.2.4.47
 Summary: A python package to simplify the usage of feature store using Teradata Vantage ...
 Author: Denis Molin
 Requires-Python: >=3.6

{tdfs4ds-0.2.4.45.dist-info → tdfs4ds-0.2.4.47.dist-info}/RECORD

@@ -2,7 +2,7 @@ tdfs/__init__.py,sha256=7AcO7uB1opRCt7t2JOHworKimfAaDeO3boRW7u9Geo8,23
 tdfs/datasets.py,sha256=-b2MPEKGki2V1M8iUcoDR9uc2krIK7u1CK-EhChvihs,985
 tdfs/feature_store.py,sha256=Honu7eOAXxP4Ivz0mRlhuNkfTDzgZl5HB1WlQUwzcZ0,31354
 tdfs/data/curves.csv,sha256=q0Tm-0yu7VMK4lHvHpgi1LMeRq0lO5gJy2Q17brKbEM,112488
-tdfs4ds/__init__.py,sha256=DkgKZAKInUoRAbE5hJ7rshmfRBolqCaJKJOOA4uA9XM,61203
+tdfs4ds/__init__.py,sha256=WsOG-nk0VyiZ7HBqxVS-xxhbyp9TjogeVF8wfCSl8Rw,61213
 tdfs4ds/datasets.py,sha256=LE4Gn0muwdyrIrCrbkE92cnafUML63z1lj5bFIIVzmc,3524
 tdfs4ds/feature_engineering.py,sha256=oVnZ2V_XNGE12LKC_fNfkrWSQZLgtYRmaf8Dispi6S4,7081
 tdfs4ds/feature_store.py,sha256=y-oItPZw6nBkBcGAceaATZbkLPTsvpk0OnpzTxYofDs,68576
@@ -17,9 +17,10 @@ tdfs4ds/dataset/dataset.py,sha256=J_fgfsVdR9zSOXrUOqyotqsUD-GlQMGyuld6ueov45w,76
 tdfs4ds/dataset/dataset_catalog.py,sha256=qxS2thDW2MvsRouSFaX1M0sX2J7IzBAYD8Yf22Tsd5k,16638
 tdfs4ds/feature_store/__init__.py,sha256=a7NPCkpTx40UR5LRErwnskpABG2Vuib7F5wUjaUGCnI,209
 tdfs4ds/feature_store/entity_management.py,sha256=9ltytv3yCTG84NZXBpb1Tlkf9pOxvrNb0MVidU4pwvE,10157
-tdfs4ds/feature_store/feature_data_processing.py,sha256=IrjoL8Q6jI5QELGxSfXu0G5IeqTEEnNJKEoY9ySZF4k,45039
+tdfs4ds/feature_store/feature_data_processing.py,sha256=Hva2GRVRygHgbpQlkpplmCyrWpRKs8FEGPmzmVjC7C0,45686
 tdfs4ds/feature_store/feature_query_retrieval.py,sha256=51c6ZNlLFiBIxNPinS8ot8bjWEIb1QV2eVg69yzVF80,35381
-tdfs4ds/feature_store/feature_store_management.py,sha256=mtPQkdMDhcOrhj9IAaH-FEP_znK53cYtEv8zXAbsigg,52123
+tdfs4ds/feature_store/feature_store_management.py,sha256=RZrjjTKjJBmc3wate8A5NsnBsqtY0be18lFx5WdcWy4,52120
+tdfs4ds/genai/__init__.py,sha256=b5HwEnBhAwHu2qYwPTBZ9wkRKL4HSl_4uHTEHf5Vn1Q,13785
 tdfs4ds/process_store/__init__.py,sha256=npHR_xju5ecGmWfYHDyteLwiU3x-cL4HD3sFK_th7xY,229
 tdfs4ds/process_store/process_followup.py,sha256=E4jgQahjhVRBbfAW3JXNLId7H5qV8ozRt-6PyAQuPzg,12583
 tdfs4ds/process_store/process_query_administration.py,sha256=AOufkJ6DFUpBiGm-6Q6Dq0Aovw31UGTscZ3Ya0ewS-0,7851
@@ -32,7 +33,7 @@ tdfs4ds/utils/lineage.py,sha256=gy5M42qy5fvdWmlohAY3WPYoqAyp5VakeEmeT1YjrJQ,3783
 tdfs4ds/utils/query_management.py,sha256=kWDeTdsYcbpV5Tyhh-8uLRWvXh16nIdXNIJ97w76aNU,4848
 tdfs4ds/utils/time_management.py,sha256=oq_vlMliLuD4hO1G2zwWbj-wv4UzGWhB6-nXIsIyiMk,31935
 tdfs4ds/utils/visualization.py,sha256=5S528KoKzzkrAdCxfy7ecyqKvAXBoibNvHwz_u5ISMs,23167
-tdfs4ds-0.2.4.45.dist-info/METADATA,sha256=s_nI3zNgPckDe6VXE--iDyrV52DxAjaeaQ6NSVBbvas,14326
-tdfs4ds-0.2.4.45.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
-tdfs4ds-0.2.4.45.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
-tdfs4ds-0.2.4.45.dist-info/RECORD,,
+tdfs4ds-0.2.4.47.dist-info/METADATA,sha256=eIdXWOBQF9N12xALV-EIJzj8tCefp_kdcV4oWpaEc8E,14326
+tdfs4ds-0.2.4.47.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
+tdfs4ds-0.2.4.47.dist-info/top_level.txt,sha256=wMyVkMvnBn8RRt1xBveGQxOpWFijPMPkMiE7G2mi8zo,8
+tdfs4ds-0.2.4.47.dist-info/RECORD,,