answer42 0.3.2__tar.gz → 0.3.4__tar.gz

{answer42-0.3.2 → answer42-0.3.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: answer42
-Version: 0.3.2
+Version: 0.3.4
 Summary: Answer42 — The Answer to Life, Universe, and 1C — UI Driver. MCP-powered 1C:Enterprise UI automation: click, fill, navigate, test, and introspect managed forms through the test-client API
 Author: Marvin (AI Assistant), 42Clouds, and contributors
 Author-email: "Kosolapov Stanislav (proDOOMman)" <prodoomman@gmail.com>

{answer42-0.3.2 → answer42-0.3.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "answer42"
-version = "0.3.2"
+version = "0.3.4"
 description = "Answer42 — The Answer to Life, Universe, and 1C — UI Driver. MCP-powered 1C:Enterprise UI automation: click, fill, navigate, test, and introspect managed forms through the test-client API"
 readme = "README.md"
 requires-python = ">=3.11"

{answer42-0.3.2 → answer42-0.3.4}/src/mcp_1c/rag/parsers.py

@@ -21,6 +21,8 @@ from .model import (
 )
 
 KIND_DIRS = {
+    "Constants": "Константа",
+    "CommonForms": "ОбщаяФорма",
     "Catalogs": "Справочник",
     "Documents": "Документ",
     "DataProcessors": "Обработка",
@@ -33,6 +35,8 @@ KIND_DIRS = {
     "CalculationRegisters": "РегистрРасчета",
 }
 
+FORM_FILE_NAMES = {"Form.form", "Form.xml"}
+
 
 def _strip_ns(tag: str) -> str:
     return tag.rsplit("}", 1)[-1]
@@ -199,7 +203,10 @@ def parse_edt_source(source: SourceInfo) -> ParsedSource:
             parsed.objects.append(MetadataObject(full, ru_kind, obj_name, synonym, str(mdo), source.format, hierarchical, [o for o in owners if o]))
             if root is not None:
                 _extract_metadata_children(parsed, root, full, mdo)
-            _extract_forms(parsed, mdo.parent, full)
+            if ru_kind == "ОбщаяФорма":
+                _extract_common_form(parsed, mdo.parent, full)
+            else:
+                _extract_forms(parsed, mdo.parent, full)
             _extract_data_composition_schemas(parsed, mdo.parent, full)
             _extract_help_pages(parsed, mdo.parent, full)
     return parsed
@@ -231,12 +238,26 @@ def parse_designer_xml_source(source: SourceInfo) -> ParsedSource:
             if root is not None:
                 _extract_metadata_children(parsed, root, full, xml)
             object_dir = xml.parent if xml.parent != base else base / xml.stem
-            _extract_forms(parsed, object_dir, full)
+            if ru_kind == "ОбщаяФорма":
+                _extract_common_form(parsed, object_dir, full)
+            else:
+                _extract_forms(parsed, object_dir, full)
             _extract_data_composition_schemas(parsed, object_dir, full)
             _extract_help_pages(parsed, object_dir, full)
     return parsed
 
 
+def _extract_common_form(parsed: ParsedSource, object_dir: Path, full: str) -> None:
+    form_xml = _find_form_file(object_dir)
+    parsed.forms.append(Form(full, "Форма", "CommonForm", str(form_xml) if form_xml.exists() else str(object_dir)))
+    if not form_xml.exists():
+        return
+    root = _xml_root(form_xml)
+    if root is not None:
+        _extract_form_elements(parsed, root, full, "Форма", form_xml)
+        _extract_embedded_data_composition_fields(parsed, root, full, "Форма", form_xml)
+
+
 def _extract_metadata_children(parsed: ParsedSource, root: ET.Element, full: str, source_path: Path) -> None:
     # Heuristic parser covering both EDT and designer XML structures.
     # Keep object attributes and table-part columns separate: XML dumps often
@@ -273,9 +294,7 @@ def _extract_forms(parsed: ParsedSource, object_dir: Path, full: str) -> None:
         return
     for form_dir in sorted([p for p in forms_dir.iterdir() if p.is_dir()]):
         form_name = form_dir.name
-        form_xml = form_dir / "Ext" / "Form.xml"
-        if not form_xml.exists():
-            form_xml = form_dir / "Form.xml"
+        form_xml = _find_form_file(form_dir)
         parsed.forms.append(Form(full, form_name, None, str(form_xml) if form_xml.exists() else str(form_dir)))
         if form_xml.exists():
             root = _xml_root(form_xml)
@@ -284,23 +303,34 @@ def _extract_forms(parsed: ParsedSource, object_dir: Path, full: str) -> None:
                 _extract_embedded_data_composition_fields(parsed, root, full, form_name, form_xml)
 
 
+def _find_form_file(form_dir: Path) -> Path:
+    for candidate_dir in (form_dir / "Ext", form_dir):
+        for name in FORM_FILE_NAMES:
+            form_file = candidate_dir / name
+            if form_file.exists():
+                return form_file
+    return form_dir / "Ext" / "Form.xml"
+
+
 def _extract_form_elements(parsed: ParsedSource, root: ET.Element, full: str, form_name: str, source_path: Path) -> None:
     for elem in root.iter():
         tag = _strip_ns(elem.tag)
         low = tag.lower()
-        if low not in {"element", "item", "items", "button", "command", "field", "table", "group"}:
+        if low not in {"element", "item", "items", "button", "command", "field", "table", "group", "extendedtooltip", "contextmenu"}:
             continue
         name = elem.attrib.get("name") or elem.attrib.get("Name") or _first_text(elem, ["name", "Name"])
         if not name:
             continue
+        title = elem.attrib.get("title") or elem.attrib.get("Title") or _first_text(elem, ["title", "Title", "caption", "Caption"])
+        data_path = _form_data_path(elem)
         parsed.form_elements.append(
             FormElement(
                 full,
                 form_name,
                 name,
                 tag,
-                elem.attrib.get("title") or elem.attrib.get("Title") or _first_text(elem, ["title", "Title", "caption", "Caption"]),
-                elem.attrib.get("dataPath") or elem.attrib.get("path") or _first_text(elem, ["dataPath", "path", "DataPath"]),
+                title,
+                data_path,
                 elem.attrib.get("commandName") or _first_text(elem, ["commandName", "CommandName"]),
                 None,
                 str(source_path),
@@ -308,6 +338,20 @@ def _extract_form_elements(parsed: ParsedSource, root: ET.Element, full: str, fo
         )
 
 
+def _form_data_path(elem: ET.Element) -> str | None:
+    direct = elem.attrib.get("dataPath") or elem.attrib.get("path") or _direct_text(elem, ["dataPath", "path", "DataPath"])
+    if direct:
+        return direct
+    for child in list(elem):
+        if _strip_ns(child.tag).lower() == "datapath":
+            segments = [_text(segment) for segment in child.iter() if _strip_ns(segment.tag).lower() in {"segments", "segment"}]
+            values = [value for value in segments if value]
+            if values:
+                return ".".join(values)
+            return _text(child)
+    return None
+
+
 def _extract_embedded_data_composition_fields(parsed: ParsedSource, root: ET.Element, full: str, form_name: str, source_path: Path) -> None:
     dataset_name: str | None = None
     for elem in root.iter():
@@ -341,17 +385,20 @@ def _extract_embedded_data_composition_fields(parsed: ParsedSource, root: ET.Ele
 
 
 def _extract_help_pages(parsed: ParsedSource, object_dir: Path, full: str) -> None:
-    help_dir = object_dir / "Help"
-    if not help_dir.exists():
-        return
-    for page in sorted(help_dir.glob("*.html")):
-        raw = page.read_text(encoding="utf-8", errors="ignore")
-        title_match = re.search(r"<title[^>]*>(.*?)</title>", raw, flags=re.IGNORECASE | re.DOTALL)
-        h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", raw, flags=re.IGNORECASE | re.DOTALL)
-        title = _html_to_text((title_match or h1_match).group(1)) if (title_match or h1_match) else None
-        text = _html_to_text(raw)
-        if text:
-            parsed.help_pages.append(HelpPage(full, page.stem, title, text, str(page), {"file": page.name}))
+    # EDT exports usually keep help as <Object>/Help/*.html, while Designer
+    # hierarchical dumps keep it under <Object>/Ext/Help/*.html with
+    # <Object>/Ext/Help.xml next to the language-specific pages.
+    for help_dir in (object_dir / "Help", object_dir / "Ext" / "Help"):
+        if not help_dir.exists():
+            continue
+        for page in sorted(help_dir.glob("*.html")):
+            raw = page.read_text(encoding="utf-8-sig", errors="ignore")
+            title_match = re.search(r"<title[^>]*>(.*?)</title>", raw, flags=re.IGNORECASE | re.DOTALL)
+            h1_match = re.search(r"<h1[^>]*>(.*?)</h1>", raw, flags=re.IGNORECASE | re.DOTALL)
+            title = _html_to_text((title_match or h1_match).group(1)) if (title_match or h1_match) else None
+            text = _html_to_text(raw)
+            if text:
+                parsed.help_pages.append(HelpPage(full, page.stem, title, text, str(page), {"file": page.name}))
 
 
 def _html_to_text(raw: str) -> str:
@@ -410,5 +457,38 @@ def navigation_for_object(full_name: str, kind: str) -> tuple[str, str] | None:
     return None
 
 
+def split_1c_identifier(text: str | None) -> str:
+    """Return a human-readable variant of 1C metadata identifiers.
+
+    1C metadata names are commonly written as CamelCase identifiers, e.g.
+    ``ОборотноСальдоваяВедомостьПоСчету``.  FTS tokenizers do not split those
+    words automatically, so business queries like ``оборотно сальдовая`` miss
+    chunks that contain only the technical identifier.  Keep this helper small
+    and deterministic: it does not add domain synonyms, it only exposes words
+    already present in the metadata name.
+    """
+    if not text:
+        return ""
+    value = re.sub(r"[._/\\:-]+", " ", text)
+    value = re.sub(r"(?<=[а-яёa-z0-9])(?=[А-ЯЁA-Z])", " ", value)
+    return re.sub(r"\s+", " ", value).strip()
+
+
+def search_text_variants(*values: str | None) -> str:
+    parts: list[str] = []
+    seen: set[str] = set()
+    for value in values:
+        if not value:
+            continue
+        for part in (value, split_1c_identifier(value)):
+            normalized = normalize_query(part)
+            if normalized and normalized not in seen:
+                seen.add(normalized)
+                parts.append(part)
+    return " ".join(parts)
+
+
 def normalize_query(text: str) -> str:
-    return re.sub(r"\s+", " ", text.strip().lower().replace("ё", "е"))
+    text = text.replace("ё", "е")
+    text = re.sub(r"[\u2010-\u2015–—−-]+", " ", text)
+    return re.sub(r"\s+", " ", text.strip().lower())

{answer42-0.3.2 → answer42-0.3.4}/src/mcp_1c/rag/service.py

@@ -1,16 +1,22 @@
 from __future__ import annotations
 
+import hashlib
+import json
+import math
 import os
+import re
 import subprocess
 from pathlib import Path
 from typing import Any
 
 from .detect import detect_source_format, discover_source_roots
 from .model import SourceInfo
-from .parsers import navigation_for_object, normalize_query, parse_source
+from .parsers import navigation_for_object, normalize_query, parse_source, search_text_variants
 from .store import RagStore, json_dumps, row_to_dict
 
 DEFAULT_RAG_DB = Path(os.environ.get("MCP_1C_RAG_DB", "build/rag/onec-rag.sqlite"))
+EMBEDDING_MODEL = "answer42-local-hash-v1"
+EMBEDDING_DIM = 256
 
 
 class RagService:
@@ -246,9 +252,11 @@ class RagService:
                     nav = navigation_for_object(obj.full_name, obj.kind)
                     if nav:
                         con.execute("INSERT INTO navigation_links(object_id, kind, url) VALUES(?,?,?)", (oid, nav[0], nav[1]))
-                    chunk_text = f"{obj.full_name} {obj.synonym or ''} {obj.kind} navigation {nav[1] if nav else ''}"
+                    object_terms = search_text_variants(obj.full_name, obj.name, obj.synonym)
+                    chunk_text = f"{obj.full_name} {object_terms} {obj.synonym or ''} {obj.kind} navigation {nav[1] if nav else ''}"
                     cid = _insert_chunk(con, snapshot_id, oid, "object", chunk_text, obj.source_path, {"source": src["name"]})
                     con.execute("INSERT INTO search_fts(text, object_full_name, kind, source_path, chunk_id) VALUES(?,?,?,?,?)", (chunk_text, obj.full_name, obj.kind, obj.source_path, cid))
+                    _insert_embedding(con, cid, chunk_text)
                     counts["chunks"] += 1
                     if obj.synonym:
                         _insert_business_term(con, snapshot_id, obj.synonym, oid, 0.7, "object synonym")
@@ -258,6 +266,12 @@ class RagService:
                         continue
                     con.execute("INSERT INTO attributes(object_id,name,type_name,synonym,required,source_path,role,raw_json) VALUES(?,?,?,?,?,?,?,?)", (oid, attr.name, attr.type_name, attr.synonym, attr.required, attr.source_path, attr.role, json_dumps(attr.raw)))
                     counts["attributes"] += 1
+                    text_parts = [attr.object_full_name, attr.role, attr.name, search_text_variants(attr.name, attr.synonym), attr.synonym or "", attr.type_name or ""]
+                    chunk_text = " ".join(part for part in text_parts if part)
+                    cid = _insert_chunk(con, snapshot_id, oid, "attribute", chunk_text, attr.source_path, {"source": src["name"], "role": attr.role})
+                    con.execute("INSERT INTO search_fts(text, object_full_name, kind, source_path, chunk_id) VALUES(?,?,?,?,?)", (chunk_text, attr.object_full_name, "Реквизит", attr.source_path, cid))
+                    _insert_embedding(con, cid, chunk_text)
+                    counts["chunks"] += 1
                 for tp in parsed.table_parts:
                     oid = object_ids.get(tp.object_full_name)
                     if not oid:
@@ -276,12 +290,25 @@ class RagService:
                     cur = con.execute("INSERT INTO forms(object_id,name,kind,source_path,raw_json) VALUES(?,?,?,?,?)", (oid, form.name, form.kind, form.source_path, json_dumps(form.raw)))
                     form_ids[(form.object_full_name, form.name)] = cur.lastrowid
                     counts["forms"] += 1
+                    text_parts = [form.object_full_name, "форма", form.name, search_text_variants(form.name), form.kind or ""]
+                    chunk_text = " ".join(part for part in text_parts if part)
+                    cid = _insert_chunk(con, snapshot_id, oid, "form", chunk_text, form.source_path, {"source": src["name"], "form": form.name})
+                    con.execute("INSERT INTO search_fts(text, object_full_name, kind, source_path, chunk_id) VALUES(?,?,?,?,?)", (chunk_text, form.object_full_name, "Форма", form.source_path, cid))
+                    _insert_embedding(con, cid, chunk_text)
+                    counts["chunks"] += 1
                 for elem in parsed.form_elements:
                     fid = form_ids.get((elem.object_full_name, elem.form_name))
                     if not fid:
                         continue
                     con.execute("INSERT INTO form_elements(form_id,name,element_type,title,data_path,command_name,parent_name,source_path,raw_json) VALUES(?,?,?,?,?,?,?,?,?)", (fid, elem.name, elem.element_type, elem.title, elem.data_path, elem.command_name, elem.parent_name, elem.source_path, json_dumps(elem.raw)))
                     counts["form_elements"] += 1
+                    oid = object_ids.get(elem.object_full_name)
+                    text_parts = [elem.object_full_name, "элемент формы", elem.form_name, elem.name, search_text_variants(elem.name, elem.title, elem.data_path), elem.title or "", elem.data_path or "", elem.command_name or "", elem.element_type or ""]
+                    chunk_text = " ".join(part for part in text_parts if part)
+                    cid = _insert_chunk(con, snapshot_id, oid, "form_element", chunk_text, elem.source_path, {"source": src["name"], "form": elem.form_name, "element": elem.name})
+                    con.execute("INSERT INTO search_fts(text, object_full_name, kind, source_path, chunk_id) VALUES(?,?,?,?,?)", (chunk_text, elem.object_full_name, "ЭлементФормы", elem.source_path, cid))
+                    _insert_embedding(con, cid, chunk_text)
+                    counts["chunks"] += 1
                 for dcs_field in parsed.data_composition_fields:
                     oid = object_ids.get(dcs_field.object_full_name)
                     if not oid:
@@ -295,15 +322,23 @@ class RagService:
                     chunk_text = " ".join(part for part in text_parts if part)
                     cid = _insert_chunk(con, snapshot_id, oid, "dcs_field", chunk_text, dcs_field.source_path, {"source": src["name"], "schema": dcs_field.schema_name})
                     con.execute("INSERT INTO search_fts(text, object_full_name, kind, source_path, chunk_id) VALUES(?,?,?,?,?)", (chunk_text, dcs_field.object_full_name, "СКД", dcs_field.source_path, cid))
+                    _insert_embedding(con, cid, chunk_text)
                     counts["chunks"] += 1
                 for help_page in parsed.help_pages:
                     oid = object_ids.get(help_page.object_full_name)
                     if not oid:
                         continue
-                    text_parts = [help_page.object_full_name, "справка", help_page.language, help_page.title or "", help_page.text or ""]
+                    object_row = con.execute("SELECT name, synonym FROM metadata_objects WHERE id=?", (oid,)).fetchone()
+                    object_terms = search_text_variants(
+                        help_page.object_full_name,
+                        object_row["name"] if object_row else None,
+                        object_row["synonym"] if object_row else None,
+                    )
+                    text_parts = [help_page.object_full_name, object_terms, "справка", help_page.language, help_page.title or "", help_page.text or ""]
                     chunk_text = "\n".join(part for part in text_parts if part)
                     cid = _insert_chunk(con, snapshot_id, oid, "help", chunk_text, help_page.source_path, {"source": src["name"], "language": help_page.language, "title": help_page.title})
                     con.execute("INSERT INTO search_fts(text, object_full_name, kind, source_path, chunk_id) VALUES(?,?,?,?,?)", (chunk_text, help_page.object_full_name, "Справка", help_page.source_path, cid))
+                    _insert_embedding(con, cid, chunk_text)
                     counts["help_pages"] += 1
                     counts["chunks"] += 1
         return {"db_path": self.db_path, **counts}
@@ -497,7 +532,45 @@ class RagService:
                         seen.add(key)
                     if len(fts_rows) >= limit:
                         break
-        return {"query": text, "snapshot": snapshot, "terms": term_rows, "fts": fts_rows}
+            semantic_rows = self._semantic_query(con, q, snapshot_id, limit)
+        return {"query": text, "snapshot": snapshot, "terms": term_rows, "fts": fts_rows, "semantic": semantic_rows}
+
+    def _semantic_query(self, con, query: str, snapshot_id: int | None, limit: int) -> list[dict[str, Any]]:
+        query_vector = _embedding_vector(query)
+        params: list[Any] = [EMBEDDING_MODEL]
+        snapshot_filter = ""
+        if snapshot_id is not None:
+            snapshot_filter = "AND rc.snapshot_id=?"
+            params.append(snapshot_id)
+        rows = con.execute(
+            f"""
+            SELECT rc.text, rc.source_path, mo.full_name AS object_full_name, mo.kind,
+                   re.vector_json
+            FROM rag_embeddings re
+            JOIN rag_chunks rc ON rc.id=re.chunk_id
+            LEFT JOIN metadata_objects mo ON mo.id=rc.object_id
+            WHERE re.model=? {snapshot_filter}
+            """,
+            params,
+        )
+        scored: list[dict[str, Any]] = []
+        for row in rows:
+            vector = json.loads(row["vector_json"])
+            score = _cosine(query_vector, vector)
+            if score <= 0:
+                continue
+            scored.append(
+                {
+                    "text": row["text"],
+                    "object_full_name": row["object_full_name"],
+                    "kind": row["kind"],
+                    "source_path": row["source_path"],
+                    "score": score,
+                    "model": EMBEDDING_MODEL,
+                }
+            )
+        scored.sort(key=lambda item: item["score"], reverse=True)
+        return scored[:limit]
 
 
 def _git_commit(path: Path) -> str | None:
@@ -530,5 +603,47 @@ def _insert_chunk(con, snapshot_id, object_id, chunk_type, text, source_path, me
     return cur.lastrowid
 
 
+def _insert_embedding(con, chunk_id: int, text: str) -> None:
+    con.execute(
+        """
+        INSERT INTO rag_embeddings(chunk_id,model,dim,vector_json,text_hash)
+        VALUES(?,?,?,?,?)
+        """,
+        (
+            chunk_id,
+            EMBEDDING_MODEL,
+            EMBEDDING_DIM,
+            json.dumps(_embedding_vector(text), separators=(",", ":")),
+            hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest(),
+        ),
+    )
+
+
+def _embedding_vector(text: str) -> list[float]:
+    vector = [0.0] * EMBEDDING_DIM
+    normalized = normalize_query(text)
+    tokens = re.findall(r"[\w]+", normalized, flags=re.UNICODE)
+    features: list[str] = []
+    for token in tokens:
+        if len(token) <= 2:
+            features.append("tok:" + token)
+            continue
+        features.append("tok:" + token)
+        for size in (3, 4):
+            if len(token) >= size:
+                features.extend(f"ng{size}:" + token[pos : pos + size] for pos in range(len(token) - size + 1))
+    for feature in features:
+        digest = hashlib.blake2b(feature.encode("utf-8"), digest_size=8).digest()
+        bucket = int.from_bytes(digest[:4], "little") % EMBEDDING_DIM
+        sign = 1.0 if digest[4] & 1 else -1.0
+        vector[bucket] += sign
+    norm = math.sqrt(sum(value * value for value in vector)) or 1.0
+    return [round(value / norm, 6) for value in vector]
+
+
+def _cosine(left: list[float], right: list[float]) -> float:
+    return sum(a * b for a, b in zip(left, right, strict=False))
+
+
 def _insert_business_term(con, snapshot_id, term, object_id, confidence, evidence) -> None:
     con.execute("INSERT INTO business_terms(snapshot_id,term,normalized_term,object_id,confidence,evidence) VALUES(?,?,?,?,?,?)", (snapshot_id, term, normalize_query(term), object_id, confidence, evidence))

{answer42-0.3.2 → answer42-0.3.4}/src/mcp_1c/rag/store.py

@@ -5,7 +5,7 @@ import sqlite3
 from pathlib import Path
 from typing import Any
 
-SCHEMA_VERSION = 3
+SCHEMA_VERSION = 4
 
 
 class RagStore:
@@ -25,7 +25,7 @@ class RagStore:
                 """
                 CREATE TABLE IF NOT EXISTS schema_info(version INTEGER NOT NULL);
                 DELETE FROM schema_info;
-                INSERT INTO schema_info(version) VALUES (3);
+                INSERT INTO schema_info(version) VALUES (4);
 
                 CREATE TABLE IF NOT EXISTS sources(
                   id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -195,6 +195,15 @@ class RagStore:
                   chunk_id UNINDEXED
                 );
 
+                CREATE TABLE IF NOT EXISTS rag_embeddings(
+                  chunk_id INTEGER PRIMARY KEY,
+                  model TEXT NOT NULL,
+                  dim INTEGER NOT NULL,
+                  vector_json TEXT NOT NULL,
+                  text_hash TEXT NOT NULL,
+                  FOREIGN KEY(chunk_id) REFERENCES rag_chunks(id) ON DELETE CASCADE
+                );
+
                 CREATE INDEX IF NOT EXISTS idx_objects_source_name ON metadata_objects(source_id, full_name);
                 CREATE INDEX IF NOT EXISTS idx_objects_kind ON metadata_objects(kind);
                 CREATE INDEX IF NOT EXISTS idx_attrs_object ON attributes(object_id);
@@ -210,6 +219,17 @@ class RagStore:
                 con.execute("ALTER TABLE attributes ADD COLUMN role TEXT NOT NULL DEFAULT 'attribute'")
 
     def clear_index_for_source(self, con: sqlite3.Connection, source_id: int) -> None:
+        con.execute(
+            """
+            DELETE FROM rag_embeddings
+            WHERE chunk_id IN (
+              SELECT rc.id
+              FROM rag_chunks rc JOIN metadata_objects mo ON mo.id=rc.object_id
+              WHERE mo.source_id=?
+            )
+            """,
+            (source_id,),
+        )
         con.execute(
             """
             DELETE FROM search_fts