nl-processing 0.1.0__tar.gz

nl_processing-0.1.0/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: nl_processing
+Version: 0.1.0
+Summary: Natural language processing playground
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic<3,>=2.0
+Requires-Dist: langchain<1,>=0.3
+Requires-Dist: langchain-openai<1,>=0.3
+Requires-Dist: opencv-python<5,>=4.10
+
+# nl_processing
+
+Minimal NLP processing project scaffolded from `proto_tg_bot` structure.
+
+## Directory Structure
+
+| Directory         | Description                         |
+|-------------------|-------------------------------------|
+| `nl_processing/`  | Main package source code            |
+| `tests/`          | Unit, integration, and e2e tests    |
+| `scripts/`        | Utility/dev scripts                 |

nl_processing-0.1.0/README.md

@@ -0,0 +1,11 @@
+# nl_processing
+
+Minimal NLP processing project scaffolded from `proto_tg_bot` structure.
+
+## Directory Structure
+
+| Directory         | Description                         |
+|-------------------|-------------------------------------|
+| `nl_processing/`  | Main package source code            |
+| `tests/`          | Unit, integration, and e2e tests    |
+| `scripts/`        | Utility/dev scripts                 |

nl_processing-0.1.0/nl_processing/core/exceptions.py

@@ -0,0 +1,10 @@
+class APIError(Exception):
+    """Wraps upstream OpenAI/LangChain API failures."""
+
+
+class TargetLanguageNotFoundError(Exception):
+    """Raised when no text in the target language is detected."""
+
+
+class UnsupportedImageFormatError(Exception):
+    """Raised when the image format is not supported by the OpenAI API."""

nl_processing-0.1.0/nl_processing/core/models.py

@@ -0,0 +1,21 @@
+from enum import Enum
+
+from pydantic import BaseModel
+
+
+class Language(Enum):
+    NL = "nl"
+    RU = "ru"
+
+
+class ExtractedText(BaseModel):
+    text: str
+
+
+class WordEntry(BaseModel):
+    normalized_form: str
+    word_type: str
+
+
+class TranslationResult(BaseModel):
+    translation: str

nl_processing-0.1.0/nl_processing/core/prompts.py

@@ -0,0 +1,98 @@
+import json
+import pathlib
+
+from langchain_core.load import load
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.runnables import RunnableSerializable
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel
+
+from nl_processing.core.models import Language
+
+
+def build_translation_chain(
+    *,
+    source_language: Language,
+    target_language: Language,
+    supported_pairs: set[tuple[str, str]],
+    prompts_dir: pathlib.Path,
+    tool_schema: type[BaseModel],
+    model: str,
+) -> RunnableSerializable:  # type: ignore[type-arg]
+    """Validate a language pair, load its prompt, and return a prompt|llm chain.
+
+    This is shared infrastructure for translation-style services that follow the
+    pattern: validate pair → load JSON prompt → bind_tools → compose chain.
+
+    Args:
+        source_language: Source language enum value.
+        target_language: Target language enum value.
+        supported_pairs: Set of (src, tgt) value strings that are allowed.
+        prompts_dir: Directory containing ``<src>_<tgt>.json`` prompt files.
+        tool_schema: Pydantic model class to bind as a tool.
+        model: OpenAI model identifier string.
+
+    Returns:
+        A ``prompt | llm`` RunnableSerializable ready for ``ainvoke()``.
+
+    Raises:
+        ValueError: If the language pair is not in *supported_pairs*.
+    """
+    pair = (source_language.value, target_language.value)
+    if pair not in supported_pairs:
+        msg = (
+            f"Unsupported language pair: "
+            f"{source_language.value} -> {target_language.value}. "
+            f"Supported pairs: {supported_pairs}"
+        )
+        raise ValueError(msg)
+
+    prompt_file = f"{source_language.value}_{target_language.value}.json"
+    prompt = load_prompt(str(prompts_dir / prompt_file))
+
+    llm = ChatOpenAI(model=model, temperature=0).bind_tools(
+        [tool_schema],
+        tool_choice=tool_schema.__name__,
+    )
+    return prompt | llm
+
+
+def load_prompt(prompt_path: str) -> ChatPromptTemplate:
+    """Load a ChatPromptTemplate from a LangChain-serialized JSON file.
+
+    The JSON file must contain the output of ``langchain_core.load.dumpd(prompt)``.
+
+    Args:
+        prompt_path: Path to the prompt JSON file in LangChain native format.
+
+    Returns:
+        A ChatPromptTemplate ready for chain composition.
+
+    Raises:
+        FileNotFoundError: If the prompt file does not exist.
+        ValueError: If the JSON file is malformed or cannot be deserialized.
+        TypeError: If the file content is not a JSON object or the deserialized
+            object is not a ChatPromptTemplate.
+    """
+    path = pathlib.Path(prompt_path)
+    if not path.exists():
+        raise FileNotFoundError(f"Prompt file not found: {prompt_path}")
+
+    try:
+        with path.open("r", encoding="utf-8") as f:
+            data = json.load(f)
+    except json.JSONDecodeError as e:
+        raise ValueError(f"Invalid JSON in prompt file {prompt_path}: {e}") from e
+
+    if not isinstance(data, dict):
+        raise TypeError(f"Prompt file must contain a JSON object, got {type(data).__name__}")
+
+    try:
+        prompt = load(data)
+    except Exception as e:
+        raise ValueError(f"Failed to deserialize ChatPromptTemplate from {prompt_path}: {e}") from e
+
+    if not isinstance(prompt, ChatPromptTemplate):
+        raise TypeError(f"Expected ChatPromptTemplate, got {type(prompt).__name__} from {prompt_path}")
+
+    return prompt

nl_processing-0.1.0/nl_processing/core/scripts/prompt_author.py

@@ -0,0 +1,59 @@
+"""Prompt authoring helper — serialize ChatPromptTemplate to JSON.
+
+Usage:
+    1. Edit the ``build_prompt()`` function below to define your prompt.
+    2. Set OUTPUT_PATH to your desired output file path.
+    3. Run: uv run python nl_processing/core/scripts/prompt_author.py
+
+The output JSON can be loaded by ``nl_processing.core.prompts.load_prompt()``.
+"""
+
+import json
+
+from langchain_core.load import dumpd
+from langchain_core.prompts import ChatPromptTemplate
+
+
+def build_prompt() -> ChatPromptTemplate:
+    """Define your prompt here. Edit this function for each prompt you author."""
+    return ChatPromptTemplate.from_messages([
+        ("system", "You are a helpful assistant. Respond in {language}."),
+        ("human", "{input}"),
+    ])
+
+
+def serialize_prompt_to_json(prompt: ChatPromptTemplate, output_path: str) -> None:
+    """Serialize a ChatPromptTemplate to JSON using LangChain native format.
+
+    Args:
+        prompt: The ChatPromptTemplate to serialize.
+        output_path: Path where to save the JSON file.
+    """
+    data = dumpd(prompt)
+    with open(output_path, "w", encoding="utf-8") as f:
+        json.dump(data, f, indent=2)
+        f.write("\n")
+
+
+def save_prompt(prompt: ChatPromptTemplate, output_path: str) -> None:
+    """Serialize a prompt to JSON and print a summary to stdout.
+
+    Intended for use in ``if __name__ == "__main__"`` blocks of prompt
+    generation scripts so each script stays DRY.
+
+    Args:
+        prompt: The ChatPromptTemplate to serialize.
+        output_path: Path where to save the JSON file.
+    """
+    serialize_prompt_to_json(prompt, output_path)
+    print(f"Prompt saved to {output_path}")  # noqa: T201
+    print(f"Messages: {len(prompt.messages)}")  # noqa: T201
+    print(f"Input variables: {prompt.input_variables}")  # noqa: T201
+
+
+OUTPUT_PATH = "output_prompt.json"
+
+
+if __name__ == "__main__":
+    _prompt = build_prompt()
+    save_prompt(_prompt, OUTPUT_PATH)

nl_processing-0.1.0/nl_processing/database/service.py

@@ -0,0 +1,2 @@
+def save_translation(storage: dict[str, str], source_text: str, translated_text: str) -> None:
+    storage[source_text] = translated_text

nl_processing-0.1.0/nl_processing/extract_text_from_image/__init__.py

@@ -0,0 +1 @@
+

nl_processing-0.1.0/nl_processing/extract_text_from_image/benchmark.py

@@ -0,0 +1,57 @@
+import re
+
+import cv2
+import numpy
+
+
+def generate_test_image(
+    text: str,
+    output_path: str,
+    *,
+    width: int = 800,
+    height: int = 200,
+    font_scale: float = 1.0,
+    thickness: int = 2,
+) -> str:
+    """Generate a synthetic test image with known text rendered on it.
+
+    Returns the output file path.
+    """
+    img = numpy.zeros((height, width, 3), dtype=numpy.uint8)
+    img.fill(255)  # white background
+
+    lines = text.split("\n")
+    y_offset = 40
+    line_height = int(40 * font_scale)
+
+    for line in lines:
+        cv2.putText(
+            img,
+            line,
+            (20, y_offset),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            font_scale,
+            (0, 0, 0),
+            thickness,
+        )
+        y_offset += line_height
+
+    success = cv2.imwrite(output_path, img)
+    if not success:
+        raise ValueError(f"Failed to write image to {output_path}")
+    return output_path
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for comparison: strip whitespace, line breaks, markdown formatting."""
+    normalized = re.sub(r"[#*_~`>\-]+", "", text)  # remove markdown chars
+    normalized = re.sub(r"\s+", " ", normalized)  # collapse whitespace
+    return normalized.strip().lower()
+
+
+def evaluate_extraction(extracted: str, ground_truth: str) -> bool:
+    """Compare extracted text against ground truth after normalization.
+
+    Returns True if exact match after normalization.
+    """
+    return normalize_text(extracted) == normalize_text(ground_truth)

nl_processing-0.1.0/nl_processing/extract_text_from_image/image_encoding.py

@@ -0,0 +1,64 @@
+import base64
+import pathlib
+
+import cv2
+import numpy
+
+from nl_processing.core.exceptions import UnsupportedImageFormatError
+
+SUPPORTED_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp"}
+
+
+def get_image_format(path: str) -> str:
+    """Return the file extension (lowercase) for the given image path."""
+    return pathlib.Path(path).suffix.lower()
+
+
+def validate_image_format(path: str) -> None:
+    """Validate that the image format is supported by OpenAI Vision API.
+
+    Raises:
+        UnsupportedImageFormatError: If the file extension is not in SUPPORTED_EXTENSIONS.
+    """
+    suffix = get_image_format(path)
+    if suffix not in SUPPORTED_EXTENSIONS:
+        msg = f"Unsupported image format '{suffix}'. Supported formats: {', '.join(sorted(SUPPORTED_EXTENSIONS))}"
+        raise UnsupportedImageFormatError(msg)
+
+
+def encode_path_to_base64(path: str) -> tuple[str, str]:
+    """Read an image file and return (base64_string, media_type).
+
+    Does NOT validate format — caller is responsible for validation.
+    """
+    suffix = get_image_format(path)
+    media_type = _suffix_to_media_type(suffix)
+    with open(path, "rb") as f:
+        image_bytes = f.read()
+    base64_string = base64.b64encode(image_bytes).decode("utf-8")
+    return base64_string, media_type
+
+
+def encode_cv2_to_base64(image: numpy.ndarray) -> tuple[str, str]:
+    """Encode an OpenCV image array to base64 PNG.
+
+    Returns (base64_string, media_type).
+    """
+    success, buffer = cv2.imencode(".png", image)
+    if not success:
+        msg = "Failed to encode image to PNG"
+        raise ValueError(msg)
+    base64_string = base64.b64encode(buffer.tobytes()).decode("utf-8")
+    return base64_string, "image/png"
+
+
+def _suffix_to_media_type(suffix: str) -> str:
+    """Convert file extension to MIME media type."""
+    mapping = {
+        ".png": "image/png",
+        ".jpg": "image/jpeg",
+        ".jpeg": "image/jpeg",
+        ".gif": "image/gif",
+        ".webp": "image/webp",
+    }
+    return mapping[suffix]

nl_processing-0.1.0/nl_processing/extract_text_from_image/prompts/generate_nl_prompt.py

@@ -0,0 +1,159 @@
+"""Generate the Dutch extraction prompt (nl.json) with few-shot examples.
+
+Usage:
+    uv run python nl_processing/extract_text_from_image/prompts/generate_nl_prompt.py
+
+This script:
+1. Generates synthetic test images and encodes real photos
+2. Encodes them to base64
+3. Builds a ChatPromptTemplate with 5 few-shot examples (HumanMessage + AIMessage + ToolMessage triplets)
+4. Serializes with dumpd() and saves to nl.json
+
+The script is the source of truth — nl.json is the generated artifact.
+Re-run this script whenever example text or image parameters change.
+"""
+
+from pathlib import Path
+import tempfile
+
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+from nl_processing.extract_text_from_image.benchmark import generate_test_image
+from nl_processing.extract_text_from_image.image_encoding import encode_path_to_base64
+
+SYSTEM_INSTRUCTION = (
+    "Je bent een tekst-extractie assistent. "
+    "Extraheer alleen de Nederlandse tekst uit het aangeboden beeld. "
+    "Behoud de originele documentstructuur als markdown "
+    "(koppen, nadruk, regelafbrekingen). "
+    "Negeer tekst in andere talen. "
+    "Retourneer alleen de geëxtraheerde tekst, zonder commentaar of uitleg."
+)
+
+EXAMPLE_1_TEXT = "De kat zit op de mat"
+EXAMPLE_1_EXPECTED = "De kat zit op de mat"
+
+EXAMPLE_2_TEXT = "Welkom bij ons\nДобро пожаловать"
+EXAMPLE_2_EXPECTED = "Welkom bij ons"
+
+EXAMPLE_3_IMAGE = Path(__file__).parent / "examples" / "dutch_handwritten_mixed.jpg"
+EXAMPLE_3_EXPECTED = (
+    "getal, het\n"
+    "getrouwd\n"
+    "niet\n"
+    "nieuw\n"
+    "mooi\n"
+    "hoog\n"
+    "baan\n"
+    "kunst\n"
+    "heel\n"
+    "leren kennen\n"
+    "eeuw\n"
+    "moe\n"
+    "vroeg\n"
+    "ver\n"
+    "daar\n"
+    "tijd\n"
+    "lezen"
+)
+
+EXAMPLE_4_IMAGE = Path(__file__).parent / "examples" / "dutch_vocabulary_wide.jpg"
+EXAMPLE_4_EXPECTED = (
+    "vandaan\n"
+    "veranderen\n"
+    "verbeteren\n"
+    "vlakbij\n"
+    "volgorde, de\n"
+    "voorbeeld, het\n"
+    "voornaam, de\n"
+    "vorm, de\n"
+    "vraag, de\n"
+    "vriendin, de\n"
+    "vrouw, de\n"
+    "wat\n"
+    "week, de\n"
+    "welkom\n"
+    "werken\n"
+    "wonen\n"
+    "woonplaats, de\n"
+    "woord, het\n"
+    "ze\n"
+    "zeggen\n"
+    "zij\n"
+    "zijn\n"
+    "zijn\n"
+    "zin, de"
+)
+
+EXAMPLE_5_TEXT = "The quick brown fox jumps over the lazy dog"
+EXAMPLE_5_EXPECTED = ""
+
+OUTPUT_PATH = Path(__file__).parent / "nl.json"
+
+
+def _generate_image_b64(text: str, *, width: int = 800, height: int = 200) -> str:
+    """Generate a synthetic image and return its base64 data URL."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        img_path = str(Path(tmpdir) / "image.png")
+        generate_test_image(text, img_path, width=width, height=height, font_scale=1.2)
+        b64, media_type = encode_path_to_base64(img_path)
+    return f"data:{media_type};base64,{b64}"
+
+
+def _encode_existing_image_b64(path: Path) -> str:
+    """Encode an existing image file and return its base64 data URL."""
+    b64, media_type = encode_path_to_base64(str(path))
+    return f"data:{media_type};base64,{b64}"
+
+
+def _make_example_human(image_data_url: str) -> HumanMessage:
+    """Create a HumanMessage with an image content block."""
+    return HumanMessage(
+        content=[
+            {"type": "image_url", "image_url": {"url": image_data_url}},
+        ]
+    )
+
+
+def _make_example_ai(expected_text: str, call_id: str) -> AIMessage:
+    """Create an AIMessage with a tool_call for ExtractedText."""
+    return AIMessage(
+        content="",
+        tool_calls=[{"name": "ExtractedText", "args": {"text": expected_text}, "id": call_id}],
+    )
+
+
+def build_prompt() -> ChatPromptTemplate:
+    """Build the Dutch extraction prompt with 5 few-shot examples."""
+    img1 = _generate_image_b64(EXAMPLE_1_TEXT)
+    img2 = _generate_image_b64(EXAMPLE_2_TEXT)
+    img3 = _encode_existing_image_b64(EXAMPLE_3_IMAGE)
+    img4 = _encode_existing_image_b64(EXAMPLE_4_IMAGE)
+    img5 = _generate_image_b64(EXAMPLE_5_TEXT)
+
+    return ChatPromptTemplate.from_messages([
+        SystemMessage(content=SYSTEM_INSTRUCTION),
+        _make_example_human(img1),
+        _make_example_ai(EXAMPLE_1_EXPECTED, "call_example_1"),
+        ToolMessage(content=EXAMPLE_1_EXPECTED, tool_call_id="call_example_1"),
+        _make_example_human(img2),
+        _make_example_ai(EXAMPLE_2_EXPECTED, "call_example_2"),
+        ToolMessage(content=EXAMPLE_2_EXPECTED, tool_call_id="call_example_2"),
+        _make_example_human(img3),
+        _make_example_ai(EXAMPLE_3_EXPECTED, "call_example_3"),
+        ToolMessage(content=EXAMPLE_3_EXPECTED, tool_call_id="call_example_3"),
+        _make_example_human(img4),
+        _make_example_ai(EXAMPLE_4_EXPECTED, "call_example_4"),
+        ToolMessage(content=EXAMPLE_4_EXPECTED, tool_call_id="call_example_4"),
+        _make_example_human(img5),
+        _make_example_ai(EXAMPLE_5_EXPECTED, "call_example_5"),
+        ToolMessage(content=EXAMPLE_5_EXPECTED, tool_call_id="call_example_5"),
+        MessagesPlaceholder(variable_name="images"),
+    ])
+
+
+if __name__ == "__main__":
+    from nl_processing.core.scripts.prompt_author import save_prompt
+
+    save_prompt(build_prompt(), str(OUTPUT_PATH))

nl_processing-0.1.0/nl_processing/extract_text_from_image/service.py

@@ -0,0 +1,83 @@
+import pathlib
+
+from langchain_core.messages import HumanMessage
+from langchain_openai import ChatOpenAI
+import numpy
+
+from nl_processing.core.exceptions import APIError, TargetLanguageNotFoundError
+from nl_processing.core.models import ExtractedText, Language
+from nl_processing.core.prompts import load_prompt
+from nl_processing.extract_text_from_image.image_encoding import (
+    encode_cv2_to_base64,
+    encode_path_to_base64,
+    validate_image_format,
+)
+
+# Resolve prompts directory relative to this file
+_PROMPTS_DIR = pathlib.Path(__file__).parent / "prompts"
+
+
+class ImageTextExtractor:
+    """Extract language-specific text from images using OpenAI Vision API.
+
+    Usage:
+        extractor = ImageTextExtractor()
+        text = await extractor.extract_from_path("image.png")
+        text = await extractor.extract_from_cv2(cv2_image)
+    """
+
+    def __init__(
+        self,
+        *,
+        language: Language = Language.NL,
+        model: str = "gpt-4.1-mini",
+        reasoning_effort: str | None = None,
+        service_tier: str | None = None,
+        temperature: float | None = 0,
+    ) -> None:
+        self._language = language
+        prompt_path = str(_PROMPTS_DIR / f"{language.value}.json")
+        prompt = load_prompt(prompt_path)
+
+        llm = ChatOpenAI(
+            model=model, service_tier=service_tier, reasoning_effort=reasoning_effort, temperature=temperature
+        ).bind_tools([ExtractedText], tool_choice=ExtractedText.__name__)
+
+        self._chain = prompt | llm
+
+    async def extract_from_path(self, path: str) -> str:
+        """Extract text from image at the given file path.
+
+        Returns markdown-formatted text in the target language.
+        """
+        validate_image_format(path)
+        base64_string, media_type = encode_path_to_base64(path)
+        return await self._aextract(base64_string, media_type)
+
+    async def extract_from_cv2(self, image: "numpy.ndarray") -> str:
+        """Extract text from OpenCV image array.
+
+        Returns markdown-formatted text in the target language.
+        """
+        base64_string, media_type = encode_cv2_to_base64(image)
+        return await self._aextract(base64_string, media_type)
+
+    async def _aextract(self, base64_string: str, media_type: str) -> str:
+        """Internal: run the extraction chain with the base64 image."""
+        human_message = HumanMessage(
+            content=[
+                {"type": "image_url", "image_url": {"url": f"data:{media_type};base64,{base64_string}"}},
+            ]
+        )
+        try:
+            response = await self._chain.ainvoke({"images": [human_message]})
+            result = ExtractedText(**response.tool_calls[0]["args"])  # type: ignore[attr-defined]
+        except Exception as e:
+            raise APIError(str(e)) from e
+
+        # Check if target language text was found
+        if not result.text.strip():
+            msg = "No text in the target language was found in the image"
+            raise TargetLanguageNotFoundError(msg)
+
+        return result.text

nl_processing-0.1.0/nl_processing/extract_words_from_text/prompts/generate_nl_prompt.py

@@ -0,0 +1,122 @@
+"""Generate the Dutch word extraction prompt (nl.json) with few-shot examples.
+
+Usage:
+    uv run python nl_processing/extract_words_from_text/prompts/generate_nl_prompt.py
+
+The script is the source of truth -- nl.json is the generated artifact.
+Re-run this script whenever prompt content changes.
+"""
+
+import json
+from pathlib import Path
+
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+SYSTEM_INSTRUCTION = (
+    "Je bent een woord-extractie assistent voor de Nederlandse taal. "
+    "Je taak is om alle Nederlandse woorden uit de aangeboden tekst te extraheren en te normaliseren.\n\n"
+    "Regels:\n"
+    "- Negeer alle markdown-opmaak (koppen, vet, cursief, lijsten) -- extraheer alleen taalkundige inhoud.\n"
+    "- Negeer tekst in andere talen dan Nederlands.\n"
+    "- Normaliseer elk woord volgens de Nederlandse regels:\n"
+    "  - Zelfstandige naamwoorden: met lidwoord (de/het), bijv. 'de fiets', 'het huis'\n"
+    "  - Werkwoorden: infinitief, bijv. 'lopen', 'hebben'\n"
+    "  - Bijvoeglijke naamwoorden: basisvorm, bijv. 'groot', 'klein'\n"
+    "  - Voorzetsels, voegwoorden, bijwoorden: basisvorm\n"
+    "  - Eigennamen (personen/merken): ongewijzigd, type 'proper_noun_person'\n"
+    "  - Eigennamen (landen): ongewijzigd, type 'proper_noun_country'\n"
+    "- Extraheer samengestelde uitdrukkingen en fraseologische constructies als enkele eenheden.\n"
+    "- Wijs een plat woordtype toe aan elk woord. Mogelijke types: "
+    "noun, verb, adjective, adverb, preposition, conjunction, pronoun, article, numeral, "
+    "proper_noun_person, proper_noun_country.\n"
+    "- Retourneer het resultaat als een lijst van WordEntry objecten in een _WordList wrapper.\n"
+    "- Als de tekst geen Nederlandse woorden bevat, retourneer dan een lege lijst."
+)
+
+TOOL_NAME = "_WordList"
+OUTPUT_PATH = Path(__file__).parent / "nl.json"
+
+_W = dict[str, str]
+
+
+def _w(form: str, wtype: str) -> _W:
+    return {"normalized_form": form, "word_type": wtype}
+
+
+# fmt: off
+EXAMPLES: list[tuple[str, list[_W]]] = [
+    # 1: Simple sentence — noun (de/het), verb, adjective
+    ("De grote kat loopt snel.", [
+        _w("de kat", "noun"), _w("groot", "adjective"),
+        _w("lopen", "verb"), _w("snel", "adverb"),
+    ]),
+    # 2: Proper nouns and prepositions
+    ("Jan woont in Nederland.", [
+        _w("Jan", "proper_noun_person"), _w("wonen", "verb"),
+        _w("in", "preposition"), _w("Nederland", "proper_noun_country"),
+    ]),
+    # 3: Compound expression
+    ("Zij gaat er vandoor met haar vriend.", [
+        _w("zij", "pronoun"), _w("ervandoor gaan", "verb"),
+        _w("met", "preposition"), _w("haar", "pronoun"), _w("de vriend", "noun"),
+    ]),
+    # 4: Non-Dutch text — empty list
+    ("The quick brown fox jumps over the lazy dog.", []),
+    # 5: Mixed markdown with various word types
+    ("# Welkom\n\nHet **kleine** kind speelt vrolijk in de tuin.", [
+        _w("welkom", "adjective"), _w("het kind", "noun"), _w("klein", "adjective"),
+        _w("spelen", "verb"), _w("vrolijk", "adverb"),
+        _w("in", "preposition"), _w("de tuin", "noun"),
+    ]),
+    # 6: Product packaging prose — brand names, adjectives as base form, plurals singularized
+    (
+        "Met De Ruijter kunt u elke dag genieten "
+        "van een breed assortiment smakelijke producten.\n"
+        "Chocoladevlokken Melk en Puur\n"
+        "Chocoladehagel Melk en Puur\n"
+        "Vruchtenhagel\nAnijshagel\nVlokfeest\n"
+        "Gestampte Muisjes\nRose en Witte Muisjes\nBlauwe en Witte Muisjes",
+        [
+            _w("met", "preposition"), _w("De Ruijter", "proper_noun_person"),
+            _w("kunnen", "verb"), _w("u", "pronoun"), _w("elk", "adjective"),
+            _w("de dag", "noun"), _w("genieten", "verb"), _w("van", "preposition"),
+            _w("een", "article"), _w("breed", "adjective"),
+            _w("het assortiment", "noun"), _w("smakelijk", "adjective"),
+            _w("het product", "noun"), _w("de chocoladevlokken", "noun"),
+            _w("de melk", "noun"), _w("en", "conjunction"), _w("puur", "adjective"),
+            _w("de chocoladehagel", "noun"), _w("de vruchtenhagel", "noun"),
+            _w("de anijshagel", "noun"), _w("het vlokfeest", "noun"),
+            _w("gestampt", "adjective"), _w("het muisje", "noun"),
+            _w("roze", "adjective"), _w("wit", "adjective"), _w("blauw", "adjective"),
+        ],
+    ),
+]
+# fmt: on
+
+
+def _make_ai(words: list[_W], call_id: str) -> AIMessage:
+    return AIMessage(
+        content="",
+        tool_calls=[{"name": TOOL_NAME, "args": {"words": words}, "id": call_id}],
+    )
+
+
+def _make_ack(words: list[_W], call_id: str) -> ToolMessage:
+    return ToolMessage(content=json.dumps({"words": words}, ensure_ascii=False), tool_call_id=call_id)
+
+
+def build_prompt() -> ChatPromptTemplate:
+    """Build the Dutch word extraction prompt with few-shot examples."""
+    messages: list = [SystemMessage(content=SYSTEM_INSTRUCTION)]
+    for i, (text, words) in enumerate(EXAMPLES, 1):
+        cid = f"call_ex_{i}"
+        messages += [HumanMessage(content=text), _make_ai(words, cid), _make_ack(words, cid)]
+    messages.append(MessagesPlaceholder(variable_name="text"))
+    return ChatPromptTemplate.from_messages(messages)
+
+
+if __name__ == "__main__":
+    from nl_processing.core.scripts.prompt_author import save_prompt
+
+    save_prompt(build_prompt(), str(OUTPUT_PATH))

nl_processing-0.1.0/nl_processing/extract_words_from_text/service.py

@@ -0,0 +1,54 @@
+import pathlib
+
+from langchain_core.messages import HumanMessage
+from langchain_openai import ChatOpenAI
+from pydantic import BaseModel
+
+from nl_processing.core.exceptions import APIError
+from nl_processing.core.models import Language, WordEntry
+from nl_processing.core.prompts import load_prompt
+
+# Resolve prompts directory relative to this file
+_PROMPTS_DIR = pathlib.Path(__file__).parent / "prompts"
+
+
+class _WordList(BaseModel):
+    """Internal wrapper: bind_tools needs a single model, output is a list."""
+
+    words: list[WordEntry]
+
+
+class WordExtractor:
+    """Extract and normalize words from markdown text.
+
+    Usage:
+        extractor = WordExtractor()
+        words = await extractor.extract(text)
+    """
+
+    def __init__(
+        self,
+        *,
+        language: Language = Language.NL,
+        model: str = "gpt-4.1-mini",
+    ) -> None:
+        self._language = language
+        prompt_path = str(_PROMPTS_DIR / f"{language.value}.json")
+        prompt = load_prompt(prompt_path)
+
+        llm = ChatOpenAI(model=model, temperature=0).bind_tools([_WordList], tool_choice=_WordList.__name__)
+        self._chain = prompt | llm
+
+    async def extract(self, text: str) -> list[WordEntry]:
+        """Extract and normalize words from the given text.
+
+        Returns a list of WordEntry objects with normalized forms and types.
+        Returns an empty list if no words in the target language are found.
+        """
+        try:
+            response = await self._chain.ainvoke({"text": [HumanMessage(content=text)]})
+            result = _WordList(**response.tool_calls[0]["args"])  # type: ignore[attr-defined]
+        except Exception as e:
+            raise APIError(str(e)) from e
+
+        return result.words

nl_processing-0.1.0/nl_processing/translate_text/prompts/generate_nl_ru_prompt.py

@@ -0,0 +1,88 @@
+"""Generate the Dutch→Russian translation prompt (nl_ru.json) with few-shot examples.
+
+Usage:
+    uv run python nl_processing/translate_text/prompts/generate_nl_ru_prompt.py
+
+This script:
+1. Defines the system instruction (in Russian) for a professional NL→RU translator
+2. Builds few-shot examples as HumanMessage + AIMessage + ToolMessage triplets
+3. Serializes with dumpd() and saves to nl_ru.json
+
+The script is the source of truth — nl_ru.json is the generated artifact.
+Re-run this script whenever examples or system instruction change.
+"""
+
+from pathlib import Path
+
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+SYSTEM_INSTRUCTION = (
+    "Вы — профессиональный переводчик с нидерландского на русский язык. "
+    "Переведите предоставленный текст естественно, сохраняя смысл близко к оригиналу. "
+    "Сохраняйте всё форматирование markdown (заголовки, жирный, курсив, списки, разрывы абзацев). "
+    "Верните только переведённый текст — без комментариев, пояснений и префиксов. "
+    "Если ввод пуст или не содержит нидерландского текста, верните пустую строку."
+)
+
+TOOL_NAME = "_TranslatedText"
+
+EXAMPLE_1_INPUT = "De zon schijnt vandaag."
+EXAMPLE_1_OUTPUT = "Сегодня светит солнце."
+
+EXAMPLE_2_INPUT = "# Welkom\n\nDit is een **belangrijk** bericht."
+EXAMPLE_2_OUTPUT = "# Добро пожаловать\n\nЭто **важное** сообщение."
+
+EXAMPLE_3_INPUT = "Wat heb je nodig:\n\n- *Melk*\n- *Brood*\n- *Kaas*"
+EXAMPLE_3_OUTPUT = "Что тебе нужно:\n\n- *Молоко*\n- *Хлеб*\n- *Сыр*"
+
+EXAMPLE_4_INPUT = ""
+EXAMPLE_4_OUTPUT = ""
+
+EXAMPLE_5_INPUT = "The quick brown fox jumps over the lazy dog."
+EXAMPLE_5_OUTPUT = ""
+
+OUTPUT_PATH = Path(__file__).parent / "nl_ru.json"
+
+
+def _make_example_ai(translated_text: str, call_id: str) -> AIMessage:
+    """Create an AIMessage with a tool_call for _TranslatedText."""
+    return AIMessage(
+        content="",
+        tool_calls=[{"name": TOOL_NAME, "args": {"text": translated_text}, "id": call_id}],
+    )
+
+
+def build_prompt() -> ChatPromptTemplate:
+    """Build the Dutch→Russian translation prompt with 5 few-shot examples."""
+    return ChatPromptTemplate.from_messages([
+        SystemMessage(content=SYSTEM_INSTRUCTION),
+        # Example 1: simple sentence
+        HumanMessage(content=EXAMPLE_1_INPUT),
+        _make_example_ai(EXAMPLE_1_OUTPUT, "call_example_1"),
+        ToolMessage(content=EXAMPLE_1_OUTPUT, tool_call_id="call_example_1"),
+        # Example 2: markdown with headings and bold
+        HumanMessage(content=EXAMPLE_2_INPUT),
+        _make_example_ai(EXAMPLE_2_OUTPUT, "call_example_2"),
+        ToolMessage(content=EXAMPLE_2_OUTPUT, tool_call_id="call_example_2"),
+        # Example 3: list + italic
+        HumanMessage(content=EXAMPLE_3_INPUT),
+        _make_example_ai(EXAMPLE_3_OUTPUT, "call_example_3"),
+        ToolMessage(content=EXAMPLE_3_OUTPUT, tool_call_id="call_example_3"),
+        # Example 4: empty input
+        HumanMessage(content=EXAMPLE_4_INPUT),
+        _make_example_ai(EXAMPLE_4_OUTPUT, "call_example_4"),
+        ToolMessage(content=EXAMPLE_4_OUTPUT, tool_call_id="call_example_4"),
+        # Example 5: non-Dutch text
+        HumanMessage(content=EXAMPLE_5_INPUT),
+        _make_example_ai(EXAMPLE_5_OUTPUT, "call_example_5"),
+        ToolMessage(content=EXAMPLE_5_OUTPUT, tool_call_id="call_example_5"),
+        # Placeholder for actual input
+        MessagesPlaceholder(variable_name="text"),
+    ])
+
+
+if __name__ == "__main__":
+    from nl_processing.core.scripts.prompt_author import save_prompt
+
+    save_prompt(build_prompt(), str(OUTPUT_PATH))

nl_processing-0.1.0/nl_processing/translate_text/service.py

@@ -0,0 +1,58 @@
+import pathlib
+
+from langchain_core.messages import HumanMessage
+from pydantic import BaseModel
+
+from nl_processing.core.exceptions import APIError
+from nl_processing.core.models import Language
+from nl_processing.core.prompts import build_translation_chain
+
+_PROMPTS_DIR = pathlib.Path(__file__).parent / "prompts"
+
+_SUPPORTED_PAIRS: set[tuple[str, str]] = {("nl", "ru")}
+
+
+class _TranslatedText(BaseModel):
+    text: str
+
+
+class TextTranslator:
+    """Translate text between languages with markdown preservation.
+
+    Usage:
+        translator = TextTranslator(
+            source_language=Language.NL,
+            target_language=Language.RU,
+        )
+        result = await translator.translate(dutch_text)
+    """
+
+    def __init__(self, *, source_language: Language, target_language: Language, model: str = "gpt-4.1-mini") -> None:
+        self._source_language = source_language
+        self._target_language = target_language
+        self._chain = build_translation_chain(
+            source_language=source_language,
+            target_language=target_language,
+            supported_pairs=_SUPPORTED_PAIRS,
+            prompts_dir=_PROMPTS_DIR,
+            tool_schema=_TranslatedText,
+            model=model,
+        )
+
+    async def translate(self, text: str) -> str:
+        """Translate text from source to target language.
+
+        Returns the translated text or empty string for empty/non-source input.
+        """
+        if not text.strip():
+            return ""
+
+        try:
+            response = await self._chain.ainvoke({"text": [HumanMessage(content=text)]})
+            result = _TranslatedText(
+                **response.tool_calls[0]["args"]  # type: ignore[attr-defined]
+            )
+        except Exception as e:
+            raise APIError(str(e)) from e
+
+        return result.text

nl_processing-0.1.0/nl_processing/translate_word/prompts/generate_nl_ru_prompt.py

@@ -0,0 +1,112 @@
+"""Generate the Dutch-to-Russian word translation prompt (nl_ru.json).
+
+Usage:
+    uv run python nl_processing/translate_word/prompts/generate_nl_ru_prompt.py
+
+This script:
+1. Builds a ChatPromptTemplate with system instruction and 4 few-shot examples
+2. Demonstrates one-to-one order-preserving word translation
+3. Serializes with dumpd() and saves to nl_ru.json
+
+The script is the source of truth -- nl_ru.json is the generated artifact.
+Re-run this script whenever prompt content changes.
+"""
+
+import json
+from pathlib import Path
+
+from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, ToolMessage
+from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
+
+SYSTEM_INSTRUCTION = (
+    "Вы — профессиональный переводчик с нидерландского языка на русский. "
+    "Вы получаете список нидерландских слов или фраз и должны перевести каждое на русский. "
+    "Верните ровно один перевод для каждого входного слова, в том же порядке. "
+    "Количество переводов в результате должно равняться количеству слов на входе. "
+    "Каждый перевод должен быть наиболее распространённым, естественным русским эквивалентом. "
+    "Если входной список пуст, верните пустой список."
+)
+
+EXAMPLE_1_INPUT = "huis\nlopen\nsnel"
+EXAMPLE_1_OUTPUT = [
+    {"translation": "дом"},
+    {"translation": "ходить"},
+    {"translation": "быстро"},
+]
+
+EXAMPLE_2_INPUT = "de kat\nhet boek\nschrijven\nmooi\nin"
+EXAMPLE_2_OUTPUT = [
+    {"translation": "кошка"},
+    {"translation": "книга"},
+    {"translation": "писать"},
+    {"translation": "красивый"},
+    {"translation": "в"},
+]
+
+EXAMPLE_3_INPUT = "er vandoor gaan\nde fiets"
+EXAMPLE_3_OUTPUT = [
+    {"translation": "сбежать"},
+    {"translation": "велосипед"},
+]
+
+EXAMPLE_4_INPUT = ""
+EXAMPLE_4_OUTPUT: list[dict[str, str]] = []
+
+OUTPUT_PATH = Path(__file__).parent / "nl_ru.json"
+
+
+def _make_example_ai(translations: list[dict[str, str]], call_id: str) -> AIMessage:
+    """Create an AIMessage with a tool_call for _TranslationBatch."""
+    return AIMessage(
+        content="",
+        tool_calls=[
+            {
+                "name": "_TranslationBatch",
+                "args": {"translations": translations},
+                "id": call_id,
+            }
+        ],
+    )
+
+
+def build_prompt() -> ChatPromptTemplate:
+    """Build the Dutch-to-Russian word translation prompt with few-shot examples."""
+    return ChatPromptTemplate.from_messages([
+        SystemMessage(content=SYSTEM_INSTRUCTION),
+        # Example 1: 3 simple words
+        HumanMessage(content=EXAMPLE_1_INPUT),
+        _make_example_ai(EXAMPLE_1_OUTPUT, "call_example_1"),
+        ToolMessage(
+            content=json.dumps({"translations": EXAMPLE_1_OUTPUT}, ensure_ascii=False),
+            tool_call_id="call_example_1",
+        ),
+        # Example 2: 5 words with articles
+        HumanMessage(content=EXAMPLE_2_INPUT),
+        _make_example_ai(EXAMPLE_2_OUTPUT, "call_example_2"),
+        ToolMessage(
+            content=json.dumps({"translations": EXAMPLE_2_OUTPUT}, ensure_ascii=False),
+            tool_call_id="call_example_2",
+        ),
+        # Example 3: compound expression
+        HumanMessage(content=EXAMPLE_3_INPUT),
+        _make_example_ai(EXAMPLE_3_OUTPUT, "call_example_3"),
+        ToolMessage(
+            content=json.dumps({"translations": EXAMPLE_3_OUTPUT}, ensure_ascii=False),
+            tool_call_id="call_example_3",
+        ),
+        # Example 4: empty list
+        HumanMessage(content=EXAMPLE_4_INPUT),
+        _make_example_ai(EXAMPLE_4_OUTPUT, "call_example_4"),
+        ToolMessage(
+            content=json.dumps({"translations": EXAMPLE_4_OUTPUT}, ensure_ascii=False),
+            tool_call_id="call_example_4",
+        ),
+        # Placeholder for actual input
+        MessagesPlaceholder(variable_name="text"),
+    ])
+
+
+if __name__ == "__main__":
+    from nl_processing.core.scripts.prompt_author import save_prompt
+
+    save_prompt(build_prompt(), str(OUTPUT_PATH))

nl_processing-0.1.0/nl_processing/translate_word/service.py

@@ -0,0 +1,67 @@
+import pathlib
+
+from langchain_core.messages import HumanMessage
+from pydantic import BaseModel
+
+from nl_processing.core.exceptions import APIError
+from nl_processing.core.models import Language, TranslationResult
+from nl_processing.core.prompts import build_translation_chain
+
+_PROMPTS_DIR = pathlib.Path(__file__).parent / "prompts"
+
+_SUPPORTED_PAIRS: set[tuple[str, str]] = {("nl", "ru")}
+
+
+class _TranslationBatch(BaseModel):
+    translations: list[TranslationResult]
+
+
+class WordTranslator:
+    """Translate word batches between languages.
+
+    Usage:
+        translator = WordTranslator(
+            source_language=Language.NL,
+            target_language=Language.RU,
+        )
+        results = await translator.translate(["huis", "lopen"])
+    """
+
+    def __init__(
+        self,
+        *,
+        source_language: Language,
+        target_language: Language,
+        model: str = "gpt-4.1-mini",
+    ) -> None:
+        self._source_language = source_language
+        self._target_language = target_language
+        self._chain = build_translation_chain(
+            source_language=source_language,
+            target_language=target_language,
+            supported_pairs=_SUPPORTED_PAIRS,
+            prompts_dir=_PROMPTS_DIR,
+            tool_schema=_TranslationBatch,
+            model=model,
+        )
+
+    async def translate(self, words: list[str]) -> list[TranslationResult]:
+        """Translate a list of words from source to target language.
+
+        Returns one TranslationResult per input word, in the same order.
+        Returns empty list for empty input (no API call).
+        """
+        if not words:
+            return []
+
+        word_text = "\n".join(words)
+
+        try:
+            response = await self._chain.ainvoke({"text": [HumanMessage(content=word_text)]})
+            result = _TranslationBatch(
+                **response.tool_calls[0]["args"]  # type: ignore[attr-defined]
+            )
+        except Exception as e:
+            raise APIError(str(e)) from e
+
+        return result.translations

nl_processing-0.1.0/nl_processing.egg-info/PKG-INFO

@@ -0,0 +1,22 @@
+Metadata-Version: 2.4
+Name: nl_processing
+Version: 0.1.0
+Summary: Natural language processing playground
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+Requires-Dist: pydantic<3,>=2.0
+Requires-Dist: langchain<1,>=0.3
+Requires-Dist: langchain-openai<1,>=0.3
+Requires-Dist: opencv-python<5,>=4.10
+
+# nl_processing
+
+Minimal NLP processing project scaffolded from `proto_tg_bot` structure.
+
+## Directory Structure
+
+| Directory         | Description                         |
+|-------------------|-------------------------------------|
+| `nl_processing/`  | Main package source code            |
+| `tests/`          | Unit, integration, and e2e tests    |
+| `scripts/`        | Utility/dev scripts                 |

nl_processing-0.1.0/nl_processing.egg-info/SOURCES.txt

@@ -0,0 +1,29 @@
+README.md
+pyproject.toml
+nl_processing/__init__.py
+nl_processing.egg-info/PKG-INFO
+nl_processing.egg-info/SOURCES.txt
+nl_processing.egg-info/dependency_links.txt
+nl_processing.egg-info/requires.txt
+nl_processing.egg-info/top_level.txt
+nl_processing/core/__init__.py
+nl_processing/core/exceptions.py
+nl_processing/core/models.py
+nl_processing/core/prompts.py
+nl_processing/core/scripts/prompt_author.py
+nl_processing/database/__init__.py
+nl_processing/database/service.py
+nl_processing/extract_text_from_image/__init__.py
+nl_processing/extract_text_from_image/benchmark.py
+nl_processing/extract_text_from_image/image_encoding.py
+nl_processing/extract_text_from_image/service.py
+nl_processing/extract_text_from_image/prompts/generate_nl_prompt.py
+nl_processing/extract_words_from_text/__init__.py
+nl_processing/extract_words_from_text/service.py
+nl_processing/extract_words_from_text/prompts/generate_nl_prompt.py
+nl_processing/translate_text/__init__.py
+nl_processing/translate_text/service.py
+nl_processing/translate_text/prompts/generate_nl_ru_prompt.py
+nl_processing/translate_word/__init__.py
+nl_processing/translate_word/service.py
+nl_processing/translate_word/prompts/generate_nl_ru_prompt.py

nl_processing-0.1.0/nl_processing.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

nl_processing-0.1.0/nl_processing.egg-info/requires.txt

@@ -0,0 +1,4 @@
+pydantic<3,>=2.0
+langchain<1,>=0.3
+langchain-openai<1,>=0.3
+opencv-python<5,>=4.10

nl_processing-0.1.0/nl_processing.egg-info/top_level.txt

@@ -0,0 +1 @@
+nl_processing

nl_processing-0.1.0/pyproject.toml

@@ -0,0 +1,31 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "nl_processing"
+version = "0.1.0"
+description = "Natural language processing playground"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+  "pydantic>=2.0,<3",
+  "langchain>=0.3,<1",
+  "langchain-openai>=0.3,<1",
+  "opencv-python>=4.10,<5",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["nl_processing*"]
+exclude = ["tests*"]
+
+[dependency-groups]
+dev = [
+  "pytest>=9.0.2,<10",
+  "pytest-asyncio>=1.3.0,<2",
+  "pytest-xdist>=3.8.0,<4",
+  "ruff>=0.15.0,<0.16",
+  "pylint>=4.0.4,<5",
+  "vulture>=2.14.0,<3",
+]

nl_processing-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+