satif-ai 0.1.1a0__tar.gz → 0.1.2__tar.gz

{satif_ai-0.1.1a0 → satif_ai-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: satif-ai
-Version: 0.1.1a0
+Version: 0.1.2
 Summary: AI Agents for Satif
 License: MIT
 Author: Bryan Djafer
@@ -13,8 +13,8 @@ Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Requires-Dist: openai-agents (>=0.0.9,<0.0.10)
-Requires-Dist: satif-sdk (>=0.1.0,<0.2.0)
-Requires-Dist: sdif-mcp (>=0.1.0,<0.2.0)
+Requires-Dist: satif-sdk (>=0.1.0,<1.0.0)
+Requires-Dist: sdif-mcp (>=0.1.0,<1.0.0)
 Description-Content-Type: text/markdown
 
 # SATIF AI

{satif_ai-0.1.1a0 → satif_ai-0.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "satif-ai"
-version = "0.1.1a0"
+version = "0.1.2"
 description = "AI Agents for Satif"
 authors = [
     {name = "Bryan Djafer", email = "bryan.djafer@syncpulse.fr"}
@@ -12,8 +12,8 @@ requires-python = ">=3.10,<4.0"
 
 [tool.poetry.dependencies]
 openai-agents = ">=0.0.9,<0.0.10"
-satif-sdk = ">=0.1.0,<0.2.0"
-sdif-mcp = ">=0.1.0,<0.2.0"
+satif-sdk = ">=0.1.0,<1.0.0"
+sdif-mcp = ">=0.1.0,<1.0.0"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0"]
@@ -28,6 +28,7 @@ satif-core = {path = "../core", develop = true}
 satif-sdk = {path = "../sdk", develop = true}
 sdif-mcp = {path = "../mcp", develop = true}
 sdif-db = {path = "../sdif", develop = true}
+ipykernel = "^6.29.5"
 
 
 

{satif_ai-0.1.1a0 → satif_ai-0.1.2}/satif_ai/adapters/tidy.py

@@ -11,11 +11,11 @@ from typing import Optional
 # MCP and Agent imports
 from agents import Agent, Runner, function_tool
 from agents.mcp.server import MCPServerStdio
-from libs.core.satif_core.types import Datasource
 from mcp import ClientSession
 
 # SATIF imports
 from satif_core.adapters.base import Adapter
+from satif_core.types import Datasource
 from satif_sdk import SDIFDatabase
 from satif_sdk.adapters.code import AdapterError, CodeAdapter
 

satif_ai-0.1.2/satif_ai/standardizers/ai_csv.py

@@ -0,0 +1,662 @@
+import contextvars
+import csv
+import json
+import logging
+import re
+from pathlib import Path
+from typing import Any, Dict, Optional
+
+import clevercsv
+from agents import Agent, Runner, function_tool
+from agents.mcp.server import MCPServerStdio
+from charset_normalizer import detect
+from mcp import ClientSession
+from satif_core.types import Datasource, SDIFPath
+from satif_sdk.standardizers.csv import (
+    DELIMITER_SAMPLE_SIZE,
+    ENCODING_SAMPLE_SIZE,
+    CSVStandardizer,
+    SkipColumnsConfig,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# TODO: maybe we want more analysis tools:
+# get empty rows or same values rows.
+
+
+# --- Agent Prompt Definition ---
+AI_CSV_PROMPT = """
+You are an expert CSV Data Standardization Agent. Your mission is to analyze a given CSV file and determine all necessary parameters and metadata so it can be correctly standardized into a well-structured SDIF table using the underlying CSVStandardizer.
+
+**CSV File Path:** {file_path}
+**Initial Guesses (Hints for you to verify or correct):**
+- Encoding: {initial_encoding}
+- Delimiter: '{initial_delimiter}'
+
+**Your Comprehensive Task:**
+
+1.  **Core Parsing Parameters:**
+    *   Determine the correct file `encoding` (string, e.g., "utf-8", "latin-1").
+    *   Determine the correct `delimiter` (string, e.g., ",", ";", "\\t").
+    *   Determine if a `has_header` row exists (boolean: true/false).
+    *   Determine `skip_rows` (integer for initial N rows OR a list of 0-based specific row indices to skip, e.g., metadata blocks, comments, empty lines, repeated headers). Ensure the list is sorted and contains unique, non-negative integers.
+
+2.  **Table Definition:**
+    *   Generate a concise, descriptive, and SQL-safe `table_name` for the data in this CSV (string, snake_case preferred, e.g., "customer_orders_2023_q4"). This name will be sanitized by the system, but try to make it good.
+    *   Optionally, generate a `table_description` (string) providing a brief semantic overview of what the entire table represents, especially if the `table_name` isn't fully self-explanatory. (e.g., "Contains quarterly sales data for all product lines."). Only provide if it adds clear value.
+
+3.  **Column Analysis and Definition:**
+    *   For **each column** you identify that should be included in the final table:
+        *   `identifier_in_csv` (string): This is how the column is found in the *raw CSV data*.
+            *   If `has_header` is true, this is the **exact original header name** from the CSV.
+            *   If `has_header` is false, this is a **string representation of the 0-based column index** (e.g., "0", "1", "2").
+        *   `final_column_name` (string): This is the desired name for the column in the SDIF database table. It **MUST** be:
+            *   Clean and descriptive.
+            *   Sanitized by you (snake_case, lowercase, no special characters besides underscore, no spaces). The system will also sanitize it, but aim for a good one.
+            *   Potentially an improved/clarified version of the original header (e.g., fixing typos, expanding abbreviations).
+        *   `description` (string, OPTIONAL): A concise semantic description of what the data in this specific column represents.
+            *   **Provide this ONLY if the `final_column_name` is not entirely self-explanatory or if the column's content is ambiguous.**
+            *   Focus on clarity and a human-understandable meaning. (e.g., for a column `order_total_usd`, a description might be "Total amount of the order in US Dollars, including taxes but excluding discounts.")
+            *   If the `final_column_name` is very clear (e.g., `customer_email_address`), a separate description is likely NOT needed. Omit the field or set to null.
+
+4.  **Final Output:**
+    *   Respond ONLY with a single JSON object containing all the determined parameters and metadata.
+    *   The JSON object MUST adhere strictly to the following structure:
+
+    ```json
+    {{
+        "table_name": "...",
+        "table_description": null, // Or string value. Null or omit if not generated.
+        "encoding": "...",
+        "delimiter": "...",
+        "has_header": true/false,
+        "skip_rows": 0, // Integer for initial N, or sorted list of 0-based indices e.g. [0, 1, 5]
+        "columns": [
+            {{
+                "identifier_in_csv": "original_header_or_index_string",
+                "final_column_name": "sanitized_snake_case_name",
+                "description": null // Or string value. Null or omit if not generated.
+            }}
+            // ... more column objects
+        ]
+    }}
+    ```
+
+**Tools Available:**
+- `read_csv_sample(encoding: str, delimiter: str, skip_initial_rows: int = 0, row_limit: int = 20, include_row_indices: bool = False)`: Reads a sample from the *beginning* of the file. Crucial for header and initial structure.
+- `read_raw_lines(encoding: str, line_limit: int = 50, start_line: int = 0)`: Reads raw lines. Useful for finding specific rows to skip (empty, repeated headers, footers) by their 0-based index.
+- `get_file_chunk(encoding: str, start_byte: int = 0, end_byte: int = 4096)`: Reads a raw chunk. Good for diagnosing encoding/delimiter issues if `read_csv_sample` returns garbled data or errors.
+
+**General Workflow Guidance:**
+1.  **Initial Probe & Core Params:** Use `read_csv_sample` with initial hints (and `include_row_indices=True`) to examine the first few rows. Verify/correct `encoding` and `delimiter`. If `read_csv_sample` reports errors or shows garbled data, use `get_file_chunk` with different encodings to diagnose. Determine `has_header` by looking at the first non-skipped row.
+2.  **Identify Skip Rows:**
+    *   If there's metadata/comments at the top, determine how many initial rows to skip and use that for `skip_rows` (integer value).
+    *   Use `read_raw_lines` to scan for other rows to skip (e.g., empty lines, comment lines, repeated headers mid-file, summary footers). Collect all 0-based indices of such rows. If you have specific indices, `skip_rows` should be a sorted list of these indices. If you only skip initial N rows, it's an integer.
+3.  **Column Identification & Definition:**
+    *   After settling `skip_rows` and `has_header`, call `read_csv_sample` again with `skip_initial_rows` set appropriately (if `skip_rows` is an int) to see the clean data rows and the header (if present).
+    *   If `has_header` is true, the first row from this clean sample gives you the `identifier_in_csv` values (original header names).
+    *   If `has_header` is false, the `identifier_in_csv` for each column will be its 0-based index as a string (e.g., "0", "1", "2", ... for as many columns as you see in the first data row).
+    *   For each column you decide to include:
+        *   Determine its `identifier_in_csv`.
+        *   Create a clean, descriptive `final_column_name` (snake_case).
+        *   If (and ONLY IF) necessary, write a `description` for that column.
+4.  **Table Naming & Description:** Based on the clean data and column names, formulate a `table_name` and, if valuable, a `table_description`.
+5.  **Construct Final JSON:** Ensure your output is ONLY the JSON object, perfectly matching the specified schema. Pay close attention to the format of `skip_rows` and how optional fields (`table_description`, column `description`) are handled (either omit the key or set its value to `null`).
+"""
+
+
+# --- Tool Context Manager ---
+class AIStandardizerToolContext:
+    def __init__(self, file_path: Path):
+        self.file_path = file_path
+        self._original_context = None
+
+    def __enter__(self):
+        global _CURRENT_AI_CSV_TOOL_CONTEXT
+        self._original_context = _CURRENT_AI_CSV_TOOL_CONTEXT.set(self)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        global _CURRENT_AI_CSV_TOOL_CONTEXT
+        if self._original_context is not None:
+            _CURRENT_AI_CSV_TOOL_CONTEXT.reset(self._original_context)
+        self._original_context = None
+
+
+_CURRENT_AI_CSV_TOOL_CONTEXT: contextvars.ContextVar[
+    Optional[AIStandardizerToolContext]
+] = contextvars.ContextVar("current_ai_csv_tool_context", default=None)
+
+
+# --- Tool Implementations (Assumed to be largely the same, ensure JSON output is robust) ---
+@function_tool
+async def read_csv_sample(
+    encoding: str,
+    delimiter: str,
+    skip_initial_rows: int | None,  # Made optional to match agent's potential calls
+    row_limit: int | None,
+    include_row_indices: bool | None,
+) -> str:
+    if skip_initial_rows is None:
+        skip_initial_rows = 0
+    if row_limit is None:
+        row_limit = 20
+    if include_row_indices is None:
+        include_row_indices = False
+
+    context = _CURRENT_AI_CSV_TOOL_CONTEXT.get()
+    if not context or not context.file_path or not context.file_path.exists():
+        return json.dumps({"error": "File path not found in tool context."})
+
+    rows = []
+    error_message = None
+    processed_row_count = 0
+    actual_skipped_count = 0
+    try:
+        with open(context.file_path, encoding=encoding, newline="") as f:
+            for i in range(skip_initial_rows):
+                try:
+                    next(f)
+                    actual_skipped_count += 1
+                except StopIteration:
+                    error_message = f"EOF reached while skipping initial {skip_initial_rows} rows (skipped {actual_skipped_count})."
+                    break
+            if error_message:
+                return json.dumps(
+                    {"error": error_message, "rows": [], "processed_row_count": 0}
+                )
+
+            reader = csv.reader(f, delimiter=delimiter)
+            current_read_index = actual_skipped_count
+            for i, row_fields in enumerate(reader):
+                if i >= row_limit:
+                    break
+                processed_row_count += 1
+                if include_row_indices:
+                    rows.append([current_read_index] + row_fields)
+                else:
+                    rows.append(row_fields)
+                current_read_index += 1
+        return json.dumps(
+            {"rows": rows, "processed_row_count": processed_row_count, "error": None}
+        )
+    except UnicodeDecodeError as e:
+        error_message = f"Encoding error: {e}. Used encoding '{encoding}'."
+    except csv.Error as e:
+        error_message = f"CSV parsing error: {e}. Used delimiter '{delimiter}'. Check if delimiter is correct."
+    except StopIteration:
+        error_message = (
+            "Reached end of file unexpectedly." if processed_row_count == 0 else None
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error in read_csv_sample tool: {e}", exc_info=True)
+        error_message = f"Unexpected error reading sample: {str(e)}"
+    return json.dumps(
+        {
+            "error": error_message,
+            "rows": rows,
+            "processed_row_count": processed_row_count,
+        }
+    )
+
+
+@function_tool
+async def read_raw_lines(
+    encoding: str, line_limit: int | None, start_line: int | None
+) -> str:
+    if line_limit is None:
+        line_limit = 50
+    if start_line is None:
+        start_line = 0
+
+    context = _CURRENT_AI_CSV_TOOL_CONTEXT.get()
+    if not context or not context.file_path or not context.file_path.exists():
+        return json.dumps({"error": "File path not found in tool context."})
+    if start_line < 0:
+        return json.dumps({"error": "start_line cannot be negative."})
+
+    lines = []
+    error_message = None
+    actual_start_line = 0
+    lines_read_count = 0
+    try:
+        with open(context.file_path, encoding=encoding, newline="") as f:
+            for i in range(start_line):
+                try:
+                    next(f)
+                    actual_start_line += 1
+                except StopIteration:
+                    error_message = f"EOF reached while skipping to start_line {start_line} (skipped {actual_start_line})."
+                    break
+            if error_message:
+                return json.dumps(
+                    {
+                        "error": error_message,
+                        "lines": [],
+                        "start_line_processed": actual_start_line,
+                        "lines_read_count": 0,
+                    }
+                )
+
+            for i, line in enumerate(f):
+                if i >= line_limit:
+                    break
+                lines.append(line.rstrip("\r\n"))
+                lines_read_count += 1
+        return json.dumps(
+            {
+                "lines": lines,
+                "start_line_processed": actual_start_line,
+                "lines_read_count": lines_read_count,
+                "error": None,
+            }
+        )
+    except UnicodeDecodeError as e:
+        error_message = f"Encoding error: {e}. Used encoding '{encoding}'."
+    except StopIteration:
+        error_message = (
+            "Reached end of file unexpectedly." if lines_read_count == 0 else None
+        )
+    except Exception as e:
+        logger.error(f"Unexpected error in read_raw_lines tool: {e}", exc_info=True)
+        error_message = f"Unexpected error reading raw lines: {str(e)}"
+    return json.dumps(
+        {
+            "error": error_message,
+            "lines": lines,
+            "start_line_processed": actual_start_line,
+            "lines_read_count": lines_read_count,
+        }
+    )
+
+
+@function_tool
+async def get_file_chunk(
+    encoding: str, start_byte: int | None, end_byte: int | None
+) -> str:
+    if start_byte is None:
+        start_byte = 0
+    if end_byte is None:
+        end_byte = 4096
+    context = _CURRENT_AI_CSV_TOOL_CONTEXT.get()
+    if not context or not context.file_path or not context.file_path.exists():
+        return json.dumps({"error": "File path not found in tool context."})
+    if start_byte < 0 or end_byte < start_byte:
+        return json.dumps({"error": "Invalid byte range specified."})
+
+    chunk_text = ""
+    error_message = None
+    bytes_read = 0
+    try:
+        with open(context.file_path, "rb") as fb:
+            file_size = context.file_path.stat().st_size
+            effective_start_byte = min(start_byte, file_size)
+            fb.seek(effective_start_byte)
+            bytes_to_read = max(0, min(end_byte, file_size) - effective_start_byte)
+            if bytes_to_read > 0:
+                chunk_bytes = fb.read(bytes_to_read)
+                bytes_read = len(chunk_bytes)
+                chunk_text = chunk_bytes.decode(encoding, errors="replace")
+            else:
+                chunk_text = ""
+        return json.dumps(
+            {
+                "chunk": chunk_text,
+                "bytes_read": bytes_read,
+                "requested_range": [start_byte, end_byte],
+                "error": None,
+            }
+        )
+    except (UnicodeDecodeError, ValueError) as e:
+        error_message = f"Failed to decode file chunk: {e}. Used encoding '{encoding}'."
+    except OSError as e:
+        error_message = f"File read error: {e}."
+    except Exception as e:
+        logger.error(f"Unexpected error in get_file_chunk tool: {e}", exc_info=True)
+        error_message = f"Unexpected error reading file chunk: {str(e)}"
+    return json.dumps(
+        {
+            "error": error_message,
+            "chunk": chunk_text,
+            "bytes_read": bytes_read,
+            "requested_range": [start_byte, end_byte],
+        }
+    )
+
+
+# --- AICSVStandardizer Class ---
+class AICSVStandardizer(CSVStandardizer):  # Inherits from the enhanced CSVStandardizer
+    def __init__(
+        self,
+        mcp_server: Optional[MCPServerStdio] = None,
+        mcp_session: Optional[ClientSession] = None,
+        llm_model: str = "gpt-4.1-2025-04-14",
+        # --- Initial Hints (Optional) ---
+        initial_delimiter: Optional[str] = None,
+        initial_encoding: Optional[str] = None,
+        # --- Base Class Args Passthrough (some will be overridden by AI) ---
+        default_skip_columns: SkipColumnsConfig = None,  # Keep for base if AI doesn't define cols
+    ):
+        super().__init__(
+            delimiter=None,  # AI will determine
+            encoding=None,  # AI will determine
+            has_header=True,  # AI will determine
+            skip_rows=0,  # AI will determine
+            skip_columns=default_skip_columns,  # Can still be a fallback
+            descriptions=None,  # AI will generate table_description
+            table_names=None,  # AI will generate table_name
+            file_configs=None,  # AI provides all config for the one file
+            column_definitions=None,  # AI will generate column definitions
+        )
+
+        self.mcp_servers = [mcp_server] if mcp_server else []
+        self.mcp_session = mcp_session
+        self.llm_model = llm_model
+        self._initial_delimiter_hint = initial_delimiter
+        self._initial_encoding_hint = initial_encoding
+        # self.generate_description from prompt structure (table_description, column descriptions)
+
+    async def _run_analysis_agent(
+        self,
+        file_path: Path,
+        initial_encoding: str,
+        initial_delimiter: str,
+    ) -> Dict[str, Any]:
+        with AIStandardizerToolContext(file_path):
+            prompt = AI_CSV_PROMPT.format(
+                file_path=str(file_path),
+                initial_encoding=initial_encoding,
+                initial_delimiter=initial_delimiter,
+            )
+            agent = Agent(
+                name="CSV Detail Analyzer Agent",
+                mcp_servers=self.mcp_servers,
+                tools=[read_csv_sample, read_raw_lines, get_file_chunk],
+                model=self.llm_model,
+            )
+            logger.info(f"Running CSV Detail Analyzer Agent for {file_path.name}...")
+            result = await Runner.run(agent, input=prompt)
+
+            if not result or not result.final_output:
+                raise RuntimeError(
+                    f"Agent execution failed or returned no output for {file_path.name}."
+                )
+            logger.info(
+                f"Agent for {file_path.name} finished. Raw output preview: {result.final_output[:500]}..."
+            )
+
+            try:
+                final_params_text = result.final_output.strip()
+                match = re.search(r"```(?:json)?(.*)```", final_params_text, re.DOTALL)
+                if match:
+                    final_params_text = match.group(1).strip()
+
+                ai_output = json.loads(final_params_text)
+
+                # --- Validate Agent Output Structure ---
+                if not isinstance(ai_output, dict):
+                    raise ValueError("Agent did not return a valid JSON object.")
+
+                required_top_keys = {
+                    "table_name",
+                    "encoding",
+                    "delimiter",
+                    "has_header",
+                    "skip_rows",
+                    "columns",
+                }
+                if not required_top_keys.issubset(ai_output.keys()):
+                    missing = required_top_keys - ai_output.keys()
+                    raise ValueError(
+                        f"Agent JSON missing required top-level keys: {missing}"
+                    )
+
+                if not isinstance(ai_output["columns"], list):
+                    raise ValueError("Agent JSON 'columns' must be a list.")
+                if not ai_output["columns"]:  # Must have at least one column defined
+                    raise ValueError("Agent JSON 'columns' list cannot be empty.")
+
+                for col_spec in ai_output["columns"]:
+                    if not isinstance(col_spec, dict):
+                        raise ValueError(
+                            f"Each item in 'columns' list must be a dictionary. Found: {type(col_spec)}"
+                        )
+                    req_col_keys = {"identifier_in_csv", "final_column_name"}
+                    if not req_col_keys.issubset(col_spec.keys()):
+                        missing_col_keys = req_col_keys - col_spec.keys()
+                        raise ValueError(
+                            f"Column spec {col_spec.get('final_column_name', 'N/A')} missing keys: {missing_col_keys}"
+                        )
+                    # Ensure description is present, even if None (or agent omits it)
+                    if "description" not in col_spec:
+                        col_spec["description"] = None
+
+                sr = ai_output["skip_rows"]
+                if not isinstance(sr, int) and not (
+                    isinstance(sr, list) and all(isinstance(i, int) for i in sr)
+                ):
+                    raise ValueError(
+                        f"Agent JSON 'skip_rows' must be an integer or list of integers, got {type(sr)}"
+                    )
+                if isinstance(sr, list):
+                    ai_output["skip_rows"] = sorted(list(set(i for i in sr if i >= 0)))
+
+                # Ensure table_description is present, even if None
+                if "table_description" not in ai_output:
+                    ai_output["table_description"] = None
+
+                logger.info(
+                    f"Agent successfully determined parameters for {file_path.name}"
+                )
+                return ai_output
+            except json.JSONDecodeError as e:
+                logger.error(
+                    f"Agent for {file_path.name} did not return valid JSON: {e}. Output: {result.final_output}",
+                    exc_info=True,
+                )
+                raise ValueError(
+                    f"Agent failed to produce valid JSON output for {file_path.name}."
+                ) from e
+            except ValueError as e:  # Catch our custom validation errors
+                logger.error(
+                    f"Invalid JSON structure or content from agent for {file_path.name}: {e}. Output: {result.final_output}",
+                    exc_info=True,
+                )
+                raise e  # Re-raise
+
+    async def standardize(
+        self,
+        datasource: Datasource,
+        output_path: SDIFPath,  # Corrected name from output_sdif
+        *,
+        overwrite: bool = False,
+        config: Optional[Dict[str, Any]] = None,
+        **kwargs,
+    ) -> Path:
+        output_path_obj = Path(output_path)
+        if isinstance(datasource, (str, Path)):
+            input_paths = [Path(datasource)]
+        elif isinstance(datasource, list) and all(
+            isinstance(p, (str, Path)) for p in datasource
+        ):
+            input_paths = [Path(p) for p in datasource]
+            if len(input_paths) > 1:
+                logger.warning(
+                    "AICSVStandardizer currently processes one CSV file at a time for detailed AI analysis. Using the first file only."
+                )
+                input_paths = [input_paths[0]]  # Process one file if multiple given
+        else:
+            raise TypeError(
+                "datasource must be a file path string/Path object or a list of such paths."
+            )
+
+        if not input_paths:
+            raise ValueError("No input datasource provided.")
+
+        input_path = input_paths[
+            0
+        ]  # We focus on a single file for this AI standardizer
+
+        if not input_path.exists() or not input_path.is_file():
+            raise FileNotFoundError(
+                f"Input CSV file not found or is not a file: {input_path}"
+            )
+
+        logger.info(f"--- AI Analysis for file: {input_path.name} ---")
+
+        # 1. Initial Guesses for AI
+        initial_encoding_guess = self._initial_encoding_hint
+        if not initial_encoding_guess:
+            try:
+                # Use base class's _detect_encoding, need an instance or make it static/helper
+                # For simplicity, re-implement or call a static version if available.
+                # Here, we simulate it for now or assume base standardizer's helper is callable.
+                with open(input_path, "rb") as fb_enc:
+                    enc_sample = fb_enc.read(ENCODING_SAMPLE_SIZE)
+                detected_enc_info = detect(enc_sample) if enc_sample else None
+                initial_encoding_guess = (
+                    detected_enc_info["encoding"]
+                    if detected_enc_info and detected_enc_info["encoding"]
+                    else "utf-8"
+                )
+                logger.info(
+                    f"Initial encoding guess (detected): {initial_encoding_guess}"
+                )
+            except Exception as e:
+                logger.warning(
+                    f"Initial encoding detection failed: {e}. Using utf-8 as fallback guess."
+                )
+                initial_encoding_guess = "utf-8"
+        else:
+            logger.info(
+                f"Using provided initial encoding hint: {initial_encoding_guess}"
+            )
+
+        initial_delimiter_guess = self._initial_delimiter_hint
+        if not initial_delimiter_guess:
+            try:
+                with open(
+                    input_path, encoding=initial_encoding_guess, errors="ignore"
+                ) as f_delim_sample:
+                    delim_sample_text = f_delim_sample.read(DELIMITER_SAMPLE_SIZE)
+                if delim_sample_text:
+                    # Simulate base class's _detect_delimiter
+                    sniffer = clevercsv.Sniffer()
+                    dialect = sniffer.sniff(delim_sample_text)
+                    initial_delimiter_guess = dialect.delimiter if dialect else ","
+                    logger.info(
+                        f"Initial delimiter guess (detected): '{initial_delimiter_guess}'"
+                    )
+                else:
+                    initial_delimiter_guess = ","  # Fallback
+                    logger.warning(
+                        f"File empty/small, defaulting delimiter guess to ',' for {input_path.name}"
+                    )
+            except Exception as e:
+                logger.warning(
+                    f"Initial delimiter detection failed ({e}). Using ',' as fallback guess for {input_path.name}."
+                )
+                initial_delimiter_guess = ","
+        else:
+            logger.info(
+                f"Using provided initial delimiter hint: '{initial_delimiter_guess}'"
+            )
+
+        # 2. Run AI Agent Analysis
+        try:
+            ai_params = await self._run_analysis_agent(
+                input_path,
+                initial_encoding_guess,
+                initial_delimiter_guess,
+            )
+        except Exception as e:
+            logger.exception(
+                f"AI Agent analysis failed critically for {input_path.name}. Aborting."
+            )
+            raise RuntimeError(f"AI analysis failed for {input_path.name}") from e
+
+        # 3. Prepare parameters for the base CSVStandardizer
+        # The AI provides parameters for a single file processing scenario.
+
+        # Column definitions for the base standardizer:
+        # Base class expects: List[Optional[Dict[str, List[Dict[str, Any]]]]]
+        # For a single file, it's List containing one Dict: [{table_name: [col_specs...]}]
+        # Or, if base class is adapted, List containing one List: [[col_specs...]]
+
+        # The AI output `ai_params["columns"]` is already in the format:
+        # [{"identifier_in_csv": ..., "final_column_name": ..., "description": ...}, ...]
+        # This is exactly what the enhanced CSVStandardizer's `_setup_columns` expects for `defined_columns_spec`
+        # when `column_definitions` is a list containing this list of specs.
+
+        ai_column_definitions = [
+            ai_params["columns"]
+        ]  # Wrap the list of col specs for the single file/table
+
+        # The base CSVStandardizer will use its own _sanitize_name for the table name from AI.
+        # We provide it via table_names list.
+        ai_table_name = [ai_params["table_name"]]
+        ai_table_description = [
+            ai_params.get("table_description")
+        ]  # List of one description
+
+        # File-specific config for the base standardizer
+        # For a single file, this will be a list containing one dictionary.
+        file_specific_config = [
+            {
+                "encoding": ai_params["encoding"],
+                "delimiter": ai_params["delimiter"],
+                "has_header": ai_params["has_header"],
+                "skip_rows": ai_params["skip_rows"],
+                # skip_columns is not used if column_definitions are provided,
+                # as column selection is implicit in the provided definitions.
+                "skip_columns": None,  # Explicitly set to None
+            }
+        ]
+
+        logger.info(f"AI determined parameters for {input_path.name}:")
+        logger.info(f"  Table Name: {ai_table_name[0]}")
+        logger.info(f"  Encoding: {file_specific_config[0]['encoding']}")
+        logger.info(f"  Delimiter: '{file_specific_config[0]['delimiter']}'")
+        logger.info(f"  Has Header: {file_specific_config[0]['has_header']}")
+        logger.info(f"  Skip Rows: {file_specific_config[0]['skip_rows']}")
+        logger.info(
+            f"  Table Description: {ai_table_description[0] if ai_table_description and ai_table_description[0] is not None else 'N/A'}"
+        )
+        logger.info(f"  Column Definitions ({len(ai_column_definitions[0])} cols):")
+        for i, c_def in enumerate(ai_column_definitions[0]):
+            logger.info(
+                f"    {i + 1}. ID in CSV: '{c_def['identifier_in_csv']}', Final Name: '{c_def['final_column_name']}', Desc: '{c_def.get('description', 'N/A')}'"
+            )
+
+        # 4. Call Base Class Standardizer Logic with AI-derived parameters
+        # We instantiate a new CSVStandardizer configured by the AI for this specific file.
+        final_processor = CSVStandardizer(
+            # These are now single-element lists because we process one file
+            table_names=ai_table_name,
+            descriptions=ai_table_description,
+            file_configs=file_specific_config,
+            column_definitions=ai_column_definitions,  # Pass the AI-generated column specs
+            # default_skip_columns from __init__ can remain as a very deep fallback if AI somehow fails for columns
+            skip_columns=self.default_skip_columns,
+        )
+
+        try:
+            # The datasource for the base standardizer is the single input_path
+            result_path = final_processor.standardize(
+                datasource=[input_path],  # Pass as a list of one
+                output_path=output_path_obj,
+                overwrite=overwrite,
+            )
+            logger.info(
+                f"AI CSV Standardization complete for {input_path.name}. Output: {result_path}"
+            )
+            return result_path
+        except Exception as e:
+            logger.exception(
+                f"Error during final standardization step using AI parameters for {input_path.name}: {e}"
+            )
+            raise RuntimeError(
+                f"Final standardization step failed for {input_path.name}."
+            ) from e