eval-ai-library 0.1.0__py3-none-any.whl

eval_lib/datagenerator/document_loader.py

@@ -0,0 +1,510 @@
+# document_loader.py
+from __future__ import annotations
+from pathlib import Path
+from typing import List
+
+from langchain_core.documents import Document
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+# LangChain loaders (оставляем существующие)
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_community.document_loaders import Docx2txtLoader
+from langchain_community.document_loaders import TextLoader
+
+import html2text
+import markdown
+
+import io
+import json
+import zipfile
+
+# ---------------------------
+# Helper functions
+# ---------------------------
+
+
+def _read_text(p: Path) -> str:
+    return p.read_text(encoding="utf-8", errors="ignore")
+
+
+def _read_bytes(p: Path) -> bytes:
+    return p.read_bytes()
+
+
+def _csv_tsv_to_text(p: Path) -> str:
+    try:
+        import pandas as pd
+        sep = "," if p.suffix.lower() == ".csv" else "\t"
+        df = pd.read_csv(str(p), dtype=str, sep=sep,
+                         encoding="utf-8", engine="python")
+        df = df.fillna("")
+        buf = io.StringIO()
+        df.to_csv(buf, index=False)
+        return buf.getvalue()
+    except Exception:
+        try:
+            return _read_text(p)
+        except Exception:
+            return ""
+
+
+def _xlsx_to_text(p: Path) -> str:
+    try:
+        import pandas as pd
+        df = pd.read_excel(str(p), dtype=str, engine="openpyxl")
+        df = df.fillna("")
+        buf = io.StringIO()
+        df.to_csv(buf, index=False)
+        return buf.getvalue()
+    except Exception:
+        return ""
+
+
+def _pptx_to_text(p: Path) -> str:
+    try:
+        from pptx import Presentation
+        prs = Presentation(str(p))
+        texts = []
+        for slide in prs.slides:
+            for shape in slide.shapes:
+                if hasattr(shape, "text") and shape.text:
+                    texts.append(shape.text)
+        return "\n".join(texts)
+    except Exception:
+        return ""
+
+
+def _json_to_text(p: Path) -> str:
+    try:
+        data = json.loads(_read_text(p))
+
+        def flatten(obj, prefix=""):
+            lines = []
+            if isinstance(obj, dict):
+                for k, v in obj.items():
+                    lines += flatten(v, f"{prefix}{k}.")
+            elif isinstance(obj, list):
+                for i, v in enumerate(obj):
+                    lines += flatten(v, f"{prefix}{i}.")
+            else:
+                lines.append(f"{prefix[:-1]}: {obj}")
+            return lines
+        return "\n".join(flatten(data))
+    except Exception:
+        return _read_text(p)
+
+
+def _yaml_to_text(p: Path) -> str:
+    try:
+        import yaml
+        data = yaml.safe_load(_read_text(p))
+        return json.dumps(data, ensure_ascii=False, indent=2)
+    except Exception:
+        return _read_text(p)
+
+
+def _xml_to_text(p: Path) -> str:
+    try:
+        from xml.etree import ElementTree as ET
+        tree = ET.parse(str(p))
+        root = tree.getroot()
+        lines = []
+
+        def walk(node, prefix=""):
+            text = (node.text or "").strip()
+            tag = node.tag
+            if text:
+                lines.append(f"{prefix}{tag}: {text}")
+            for child in node:
+                walk(child, prefix + tag + ".")
+        walk(root)
+        return "\n".join(lines)
+    except Exception:
+        return _read_text(p)
+
+
+def _rtf_to_text(p: Path) -> str:
+    try:
+        from striprtf.striprtf import rtf_to_text
+        return rtf_to_text(_read_text(p))
+    except Exception:
+        return ""
+
+
+def _odt_to_text(p: Path) -> str:
+    try:
+        with zipfile.ZipFile(str(p)) as z:
+            from xml.etree import ElementTree as ET
+            with z.open("content.xml") as f:
+                tree = ET.parse(f)
+                root = tree.getroot()
+                texts = []
+                for elem in root.iter():
+                    if elem.text and elem.text.strip():
+                        texts.append(elem.text.strip())
+                return "\n".join(texts)
+    except Exception:
+        return ""
+
+# ---------------------------
+# PDF: LangChain -> pypdf -> PyMuPDF -> OCR(PyMuPDF+pytesseract)
+# ---------------------------
+
+
+def _pdf_text_pypdf(p: Path) -> str:
+    try:
+        from pypdf import PdfReader  # <- именно pypdf
+        reader = PdfReader(str(p))
+        texts = []
+        for page in reader.pages:
+            t = page.extract_text() or ""
+            if t.strip():
+                texts.append(t)
+        return "\n".join(texts)
+    except Exception:
+        return ""
+
+
+def _pdf_text_pymupdf(p: Path) -> str:
+    try:
+        import fitz  # PyMuPDF
+        text_parts = []
+        with fitz.open(str(p)) as doc:
+            for page in doc:
+                t = page.get_text("text") or ""
+                if t.strip():
+                    text_parts.append(t)
+        return "\n".join(text_parts)
+    except Exception:
+        return ""
+
+
+def _pdf_ocr_via_pymupdf(p: Path) -> str:
+    """Render pages via PyMuPDF and OCR pytesseract. Will work if pytesseract + tesseract are installed."""
+    try:
+        import fitz  # PyMuPDF
+        import pytesseract
+        from PIL import Image
+        import io as _io
+
+        texts = []
+        zoom = 2.0
+        mat = fitz.Matrix(zoom, zoom)
+        with fitz.open(str(p)) as doc:
+            for page in doc:
+                pix = page.get_pixmap(matrix=mat, alpha=False)
+                img = Image.open(_io.BytesIO(pix.tobytes("png")))
+                t = pytesseract.image_to_string(img) or ""
+                if t.strip():
+                    texts.append(t)
+        return "\n".join(texts)
+    except Exception:
+        return ""
+
+# ---------------------------
+# Images (OCR)
+# ---------------------------
+
+
+def _ocr_image_bytes(img_bytes: bytes) -> str:
+    try:
+        import pytesseract
+        from PIL import Image
+        import io as _io
+        img = Image.open(_io.BytesIO(img_bytes))
+        return pytesseract.image_to_string(img) or ""
+    except Exception:
+        return ""
+
+# ---------------------------
+# Docx
+# ---------------------------
+
+
+def _docx_to_text_python_docx(p: Path) -> str:
+    try:
+        import docx  # python-docx
+        d = docx.Document(str(p))
+        parts = []
+        for para in d.paragraphs:
+            if para.text:
+                parts.append(para.text)
+        # захватим текст из таблиц
+        for tbl in d.tables:
+            for row in tbl.rows:
+                cells = [cell.text for cell in row.cells]
+                if any(c.strip() for c in cells):
+                    parts.append("\t".join(cells))
+        return "\n".join(parts)
+    except Exception:
+        return ""
+
+
+def _docx_to_text_mammoth(p: Path) -> str:
+    try:
+        import mammoth
+        with open(str(p), "rb") as f:
+            result = mammoth.extract_raw_text(f)
+            return (result.value or "").strip()
+    except Exception:
+        return ""
+
+
+def _docx_to_text_zipxml(p: Path) -> str:
+    """Без зависимостей: читаем word/document.xml и вытаскиваем все w:t."""
+    try:
+        import zipfile
+        from xml.etree import ElementTree as ET
+        texts = []
+        with zipfile.ZipFile(str(p)) as z:
+            # основной документ
+            if "word/document.xml" in z.namelist():
+                with z.open("word/document.xml") as f:
+                    root = ET.parse(f).getroot()
+                    for el in root.iter():
+                        tag = el.tag.rsplit("}", 1)[-1]  # убрать namespace
+                        if tag == "t" and el.text and el.text.strip():
+                            texts.append(el.text.strip())
+            # заголовки/футеры тоже могут содержать текст
+            for name in z.namelist():
+                if name.startswith("word/header") and name.endswith(".xml"):
+                    with z.open(name) as f:
+                        root = ET.parse(f).getroot()
+                        for el in root.iter():
+                            tag = el.tag.rsplit("}", 1)[-1]
+                            if tag == "t" and el.text and el.text.strip():
+                                texts.append(el.text.strip())
+                if name.startswith("word/footer") and name.endswith(".xml"):
+                    with z.open(name) as f:
+                        root = ET.parse(f).getroot()
+                        for el in root.iter():
+                            tag = el.tag.rsplit("}", 1)[-1]
+                            if tag == "t" and el.text and el.text.strip():
+                                texts.append(el.text.strip())
+        return "\n".join(texts)
+    except Exception:
+        return ""
+
+
+def _doc_to_text_textract(p: Path) -> str:
+    """Для старого .doc. Работает, если установлен textract и системные бинарники (antiword/catdoc)."""
+    try:
+        import textract
+        return textract.process(str(p)).decode("utf-8", errors="ignore")
+    except Exception:
+        return ""
+
+# ---------------------------
+# General functions (extended)
+# ---------------------------
+
+
+def load_documents(file_paths: List[str]) -> List[Document]:
+    documents: List[Document] = []
+
+    for path in map(Path, file_paths):
+        ext = path.suffix.lower()
+
+        try:
+            # ---- PDF ----
+            if ext == ".pdf":
+                used_langchain = False
+                # 1) LangChain PyPDFLoader
+                try:
+                    docs = PyPDFLoader(str(path)).load()
+                    if docs and any((d.page_content or "").strip() for d in docs):
+                        documents += docs
+                        used_langchain = True
+                except Exception:
+                    used_langchain = False
+
+                if not used_langchain:
+                    # 2) pypdf
+                    text = _pdf_text_pypdf(path)
+                    if not text.strip():
+                        # 3) PyMuPDF
+                        text = _pdf_text_pymupdf(path)
+                    if not text.strip():
+                        # 4) OCR через PyMuPDF
+                        text = _pdf_ocr_via_pymupdf(path)
+
+                    if text.strip():
+                        documents.append(Document(page_content=text, metadata={
+                                         "source": str(path), "filetype": "pdf"}))
+                    else:
+                        print(
+                            f"⚠️  PDF has no extractable text (maybe scanned): {path.name}")
+
+            # ---- DOCX ----
+            elif ext == ".docx":
+                # 1) Пытаемся стандартным Docx2txtLoader
+                added = False
+                try:
+                    docs = Docx2txtLoader(str(path)).load()
+                    # Docx2txt иногда возвращает Document с пустым page_content
+                    docs = [d for d in docs if (d.page_content or "").strip()]
+                    if docs:
+                        documents += docs
+                        added = True
+                except Exception:
+                    added = False
+
+                if not added:
+                    # 2) python-docx
+                    text = _docx_to_text_python_docx(path)
+                    if not text.strip():
+                        # 3) mammoth
+                        text = _docx_to_text_mammoth(path)
+                    if not text.strip():
+                        # 4) zip+xml fallback
+                        text = _docx_to_text_zipxml(path)
+
+                    if text.strip():
+                        documents.append(Document(
+                            page_content=text,
+                            metadata={"source": str(path), "filetype": "docx"}
+                        ))
+                    else:
+                        print(f"⚠️  DOCX produced no text: {path.name}")
+
+            elif ext == ".doc":
+                # старый формат
+                text = _doc_to_text_textract(path)
+                if text.strip():
+                    documents.append(Document(
+                        page_content=text,
+                        metadata={"source": str(path), "filetype": "doc"}
+                    ))
+                else:
+                    print(
+                        f"⚠️  .DOC not extractable (install textract/antiword?): {path.name}")
+
+            # ---- TXT ----
+            elif ext == ".txt":
+                documents += TextLoader(str(path), encoding="utf-8").load()
+
+            # ---- HTML ----
+            elif ext in (".html", ".htm"):
+                html = _read_text(path)
+                text = html2text.html2text(html)
+                documents.append(Document(page_content=text, metadata={
+                                 "source": str(path), "filetype": "html"}))
+
+            # ---- Markdown ----
+            elif ext == ".md":
+                md = _read_text(path)
+                html = markdown.markdown(md)
+                text = html2text.html2text(html)
+                documents.append(Document(page_content=text, metadata={
+                                 "source": str(path), "filetype": "md"}))
+
+            # ---- CSV / TSV ----
+            elif ext in (".csv", ".tsv"):
+                text = _csv_tsv_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": ext.lstrip(".")}))
+                else:
+                    print(f"⚠️  Empty CSV/TSV: {path.name}")
+
+            # ---- XLSX ----
+            elif ext == ".xlsx":
+                text = _xlsx_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "xlsx"}))
+                else:
+                    print(f"⚠️  Empty XLSX: {path.name}")
+
+            # ---- PPTX ----
+            elif ext == ".pptx":
+                text = _pptx_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "pptx"}))
+                else:
+                    print(f"⚠️  Empty PPTX: {path.name}")
+
+            # ---- JSON ----
+            elif ext == ".json":
+                text = _json_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "json"}))
+                else:
+                    print(f"⚠️  Empty JSON: {path.name}")
+
+            # ---- YAML / YML ----
+            elif ext in (".yaml", ".yml"):
+                text = _yaml_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "yaml"}))
+                else:
+                    print(f"⚠️  Empty YAML: {path.name}")
+
+            # ---- XML ----
+            elif ext == ".xml":
+                text = _xml_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "xml"}))
+                else:
+                    print(f"⚠️  Empty XML: {path.name}")
+
+            # ---- RTF ----
+            elif ext == ".rtf":
+                text = _rtf_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "rtf"}))
+                else:
+                    print(f"⚠️  Empty RTF: {path.name}")
+
+            # ---- ODT ----
+            elif ext == ".odt":
+                text = _odt_to_text(path)
+                if text.strip():
+                    documents.append(Document(page_content=text, metadata={
+                                     "source": str(path), "filetype": "odt"}))
+                else:
+                    print(f"⚠️  Empty ODT: {path.name}")
+
+            # ---- Изображения (OCR) ----
+            elif ext in (".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"):
+                txt = _ocr_image_bytes(_read_bytes(path))
+                if txt.strip():
+                    documents.append(Document(page_content=txt, metadata={
+                                     "source": str(path), "filetype": "image"}))
+                else:
+                    print(f"⚠️  Image has no OCR text: {path.name}")
+
+            else:
+                print(f"⚠️  Unsupported format: {path.name} — skipped")
+
+        except Exception as exc:
+            print(f"❌ Error reading {path.name}: {exc}")
+
+    return documents
+
+
+def chunk_documents(
+    docs: List[Document],
+    chunk_size: int = 1024,
+    chunk_overlap: int = 100,
+) -> List[Document]:
+
+    splitter = RecursiveCharacterTextSplitter(
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+        separators=["\n\n", "\n", " ", ""],
+    )
+
+    chunks: List[Document] = []
+    for doc in docs:
+        for i, chunk_text in enumerate(splitter.split_text(doc.page_content)):
+            meta = dict(doc.metadata)
+            meta.update({"chunk_index": i})  # FIX
+            chunks.append(Document(page_content=chunk_text, metadata=meta))
+
+    return chunks

eval_lib/datagenerator/prompts.py

@@ -0,0 +1,192 @@
+def get_question_style_guidance(question_openness: str, question_length: str) -> str:
+    """Generate guidance based on question style parameters."""
+    guidance = []
+
+    if question_openness == "open":
+        guidance.append(
+            "- Favor open-ended questions that allow for detailed, explanatory responses")
+        guidance.append(
+            "- Include 'how', 'why', 'explain', 'describe' type questions")
+    elif question_openness == "closed":
+        guidance.append(
+            "- Focus on specific, factual questions with definitive answers")
+        guidance.append(
+            "- Include yes/no questions, specific data requests, and factual lookups")
+    else:  # mixed
+        guidance.append("- Mix both open-ended and closed questions")
+        guidance.append(
+            "- Balance exploratory questions with specific factual queries")
+
+    if question_length == "short":
+        guidance.append("- Keep inputs concise and direct (1-2 sentences)")
+    elif question_length == "long":
+        guidance.append(
+            "- Include detailed context and background in inputs (3+ sentences)")
+        guidance.append(
+            "- Provide scenarios with multiple parts or complex requirements")
+    else:  # mixed
+        guidance.append(
+            "- Vary input length from brief queries to detailed scenarios")
+
+    return "\n".join(guidance) if guidance else ""
+
+
+def get_trap_guidance(trap_density: float) -> str:
+    """Generate guidance for trap questions based on density."""
+    if trap_density == 0:
+        return "**No Trap Questions:** All questions should be answerable using the provided information."
+
+    trap_percentage = int(trap_density * 100)
+
+    return f"""**Trap Questions ({trap_percentage}% of total):**
+Create realistic scenarios where:
+- The user asks about information NOT present in the reference material
+- Questions contain subtle factual errors or misconceptions
+- Requests involve information that would require knowledge beyond the provided context
+- Make traps subtle and realistic - they should feel like genuine user mistakes or knowledge gaps
+- The agent should be able to politely indicate the limitation or correct the misconception"""
+
+
+def dataset_generation_prompt(
+    chunk: str,
+    rows_per_chunk: int,
+    agent_description: str,
+    input_format: str,
+    expected_output_format: str,
+    test_types: list[str],
+    question_length: str,
+    question_openness: str,
+    trap_density: float,
+    language: str
+) -> str:
+    """
+    Create a more realistic and detailed prompt for dataset generation.
+    """
+
+    question_style_guidance = get_question_style_guidance(
+        question_openness, question_length)
+
+    trap_guidance = get_trap_guidance(trap_density)
+
+    prompt = f"""You are an expert test case designer creating realistic evaluation scenarios for an AI agent.
+
+**Agent Context:**
+{agent_description}
+
+**Input Format:** {input_format}
+**Expected Output Format:** {expected_output_format}
+
+**Reference Information:**
+\"\"\"{chunk}\"\"\"
+
+**Your Task:**
+Create {rows_per_chunk} realistic test cases that would naturally occur when evaluating this agent. Each test case should:
+
+1. **Reflect Real User Interactions:** Generate inputs that actual users would provide in real scenarios
+2. **Be Contextually Relevant:** Use the reference information naturally, as if a user discovered it and wants to interact with the agent about it
+3. **Vary in Complexity:** Include both straightforward and nuanced scenarios
+4. **Test Different Capabilities:** Cover various aspects of the agent's expected functionality
+
+**Test Types Available:** {', '.join(test_types)}
+
+**Question Characteristics:**
+- **Length:** {question_length}
+- **Style:** {question_openness}
+{question_style_guidance}
+
+**Quality Requirements:**
+- Use natural, conversational language that real users would employ
+- Avoid academic or artificial phrasing
+- Include context and background when users would naturally provide it
+- Make questions specific enough to have clear, verifiable answers
+- Ensure variety in question types and complexity levels
+
+{trap_guidance}
+
+**Output Format:**
+Generate exactly {rows_per_chunk} JSON objects in an array. Each object must have:
+- "input": The user's natural input/question
+- "expected_output": The ideal agent response
+- "test_type": One of {test_types}
+- "is_trap": boolean indicating if this is a trap question
+
+**Language:** {language}
+
+Return ONLY the JSON array, no additional text or formatting."""
+
+    return prompt
+
+
+def dataset_generation_from_scratch_prompt(
+    max_rows: int,
+    agent_description: str,
+    input_format: str,
+    expected_output_format: str,
+    test_types: list[str],
+    question_length: str,
+    question_openness: str,
+    trap_density: float,
+    language: str
+) -> str:
+    """
+    Create a realistic prompt for generating dataset from scratch without reference material.
+    """
+
+    question_style_guidance = get_question_style_guidance(
+        question_openness, question_length)
+    trap_guidance = get_trap_guidance(trap_density)
+
+    prompt = f"""You are an expert test case designer creating comprehensive evaluation scenarios for an AI agent.
+
+**Agent Context:**
+{agent_description}
+
+**Input Format:** {input_format}
+**Expected Output Format:** {expected_output_format}
+
+**Your Task:**
+Create {max_rows} diverse, realistic test cases that thoroughly evaluate this agent's capabilities. Design scenarios that would naturally occur in real-world usage.
+
+**Test Design Principles:**
+
+1. **Real-World Scenarios:** Create inputs that actual users would provide in genuine situations
+2. **Comprehensive Coverage:** Test different aspects of the agent's functionality and knowledge domains
+3. **Varied Complexity:** Include simple queries, moderate challenges, and complex multi-step scenarios
+4. **Edge Cases:** Include boundary conditions and unusual but valid requests
+5. **Common Use Cases:** Focus on frequent user interaction patterns
+
+**Test Types Available:** {', '.join(test_types)}
+
+**Question Characteristics:**
+- **Length:** {question_length}
+- **Style:** {question_openness}
+{question_style_guidance}
+
+**Quality Requirements:**
+- Use natural, conversational language that real users would employ
+- Avoid repetitive patterns or artificial academic phrasing
+- Include appropriate context when users would naturally provide it
+- Create specific, testable scenarios with clear success criteria
+- Ensure broad coverage of the agent's expected capabilities
+- Make each test case unique and valuable for evaluation
+
+{trap_guidance}
+
+**Diversity Guidelines:**
+- Vary question topics and domains relevant to the agent
+- Include different user personas and interaction styles
+- Test both common workflows and edge cases
+- Balance straightforward requests with more complex scenarios
+
+**Output Format:**
+Generate exactly {max_rows} JSON objects in an array. Each object must have:
+- "input": The user's natural input/question
+- "expected_output": The ideal agent response
+- "test_type": One of {test_types}
+- "is_trap": boolean indicating if this is a trap question
+
+**Language:** {language}
+
+Return ONLY the JSON array, no additional text or formatting."""
+
+    return prompt