tactus 0.31.0__py3-none-any.whl → 0.34.1__py3-none-any.whl

tactus/stdlib/tac/tactus/extract/index.md

@@ -0,0 +1,96 @@
+# Extraction Module
+
+The `tactus.extract` module provides structured data extraction from unstructured text using LLM-based analysis.
+
+## Overview
+
+All extractors extend `BaseExtractor` and share a common interface. This enables consistent usage patterns and makes it easy to add new extraction strategies in the future.
+
+## When to Use
+
+- **LLMExtractor**: Use when you need to extract structured fields from natural language text. Ideal for forms, documents, conversations, and any unstructured data where field values aren't in a predictable format.
+
+## Architecture
+
+The module uses a proper Lua class hierarchy:
+
+- `BaseExtractor` - Abstract base with common interface and field validation
+- `LLMExtractor` - LLM-powered extraction with automatic retry logic
+
+All extractors return a consistent result format:
+
+```lua
+{
+    fields = {              -- Extracted field values
+        name = "John Smith",
+        age = 34,
+        email = "john@example.com"
+    },
+    retry_count = 0,        -- Number of retries needed
+    raw_response = "...",   -- LLM response (LLM only)
+    error = nil,            -- Error message if failed
+    validation_errors = {}  -- List of validation errors
+}
+```
+
+## Loading the Module
+
+```lua
+-- Load the main module
+local extract = require("tactus.extract")
+
+-- Or load specific extractors (dependencies auto-load)
+local LLMExtractor = require("tactus.extract.llm")
+```
+
+## Field Types
+
+LLMExtractor supports these field types for validation:
+
+| Type | Description | Example |
+|------|-------------|---------|
+| `string` | Text values | `"John Smith"` |
+| `number` | Numeric values (float) | `34.5` |
+| `integer` | Whole numbers | `34` |
+| `boolean` | True/false values | `true` |
+| `list`/`array` | JSON arrays | `["a", "b", "c"]` |
+| `object`/`dict` | JSON objects | `{key = "value"}` |
+
+## Performance Notes
+
+- LLM extraction typically takes 1-3 seconds per call
+- Retry logic adds latency for malformed responses
+- Consider caching extraction results for repeated operations
+- Use the `strict` parameter to control validation behavior
+
+## Extending Extractors
+
+You can extend `BaseExtractor` to create custom extractors:
+
+```lua
+local base = require("tactus.extract.base")
+local class = base.class
+local BaseExtractor = base.BaseExtractor
+
+MyExtractor = class(BaseExtractor)
+
+function MyExtractor:init(config)
+    BaseExtractor.init(self, config)
+    -- Your initialization
+end
+
+function MyExtractor:extract(text)
+    -- Your extraction logic
+    local fields = {}
+    -- ... populate fields ...
+
+    -- Validate against schema
+    local validated, errors = self:validate_fields(fields, self.fields)
+
+    return {
+        fields = validated,
+        validation_errors = errors,
+        retry_count = 0
+    }
+end
+```

tactus/stdlib/tac/tactus/extract/init.tac

@@ -0,0 +1,27 @@
+-- Tactus Extraction Module
+--
+-- Provides structured data extraction from text with:
+-- - LLM-based extraction (tactus.extract.llm)
+-- - Field validation and type coercion
+-- - Extensible base class (tactus.extract.base)
+--
+-- Usage:
+--   local extract = require("tactus.extract")
+--   local extractor = extract.LLMExtractor:new{...}
+--
+-- Or load specific extractors:
+--   local LLMExtractor = require("tactus.extract.llm")
+
+-- Load all submodules
+local base = require("tactus.extract.base")
+local llm = require("tactus.extract.llm")
+
+-- Re-export all classes
+return {
+    -- Core classes
+    BaseExtractor = base.BaseExtractor,
+    LLMExtractor = llm.LLMExtractor,
+
+    -- Helper for users who want to extend
+    class = base.class,
+}

tactus/stdlib/tac/tactus/extract/llm.tac

@@ -0,0 +1,201 @@
+-- LLM-Based Extraction
+--
+-- Provides LLM-powered structured data extraction with:
+-- - Retry logic for invalid responses
+-- - JSON parsing and validation
+-- - Field type validation
+-- - Conversational feedback for self-correction
+
+-- Load dependencies
+local base = require("tactus.extract.base")
+local BaseExtractor = base.BaseExtractor
+local class = base.class
+local json = require("tactus.io.json")
+
+-- ============================================================================
+-- LLMExtractor
+-- ============================================================================
+
+local LLMExtractor = class(BaseExtractor)
+
+function LLMExtractor:init(config)
+    BaseExtractor.init(self, config)
+
+    -- Validate required fields
+    assert(config.fields, "LLMExtractor requires 'fields' field")
+    assert(config.prompt, "LLMExtractor requires 'prompt' field")
+
+    self.fields = config.fields
+    self.prompt = config.prompt
+    self.max_retries = config.max_retries or 3
+    self.temperature = config.temperature or 0.3
+    self.model = config.model
+    self.strict = config.strict ~= false  -- Default to strict mode
+
+    -- Build extraction system prompt
+    self.system_prompt = self:build_system_prompt()
+
+    -- Create agent
+    local agent_config = {
+        system_prompt = self.system_prompt,
+        temperature = self.temperature,
+    }
+
+    if self.model then
+        local provider, model_id = self.model:match("([^/]+)/(.+)")
+        if provider and model_id then
+            agent_config.provider = provider
+            agent_config.model = model_id
+        end
+    end
+
+    self.agent = Agent(agent_config)
+end
+
+function LLMExtractor:build_system_prompt()
+    -- Build fields description
+    local fields_lines = {}
+    for name, type_ in pairs(self.fields) do
+        table.insert(fields_lines, string.format("  - %s: %s", name, type_))
+    end
+    local fields_description = table.concat(fields_lines, "\n")
+
+    return string.format([[You are an information extraction assistant. Your task is to extract structured data according to the following instruction:
+
+%s
+
+FIELDS TO EXTRACT:
+%s
+
+IMPORTANT RULES:
+1. You MUST respond with a valid JSON object containing the extracted fields.
+2. Include ONLY the specified fields in your response.
+3. Use null for fields that cannot be extracted from the input.
+4. For "number" fields, return numeric values (not strings).
+5. For "list" fields, return JSON arrays.
+6. For "boolean" fields, return true or false.
+7. Do NOT include any explanation or text outside the JSON.
+
+RESPONSE FORMAT:
+{
+  "field1": "extracted value",
+  "field2": 123,
+  ...
+}]], self.prompt, fields_description)
+end
+
+function LLMExtractor:parse_json(response)
+    if not response or response == "" then
+        return nil, {"Empty response"}
+    end
+
+    -- Try to find JSON object in response
+    local json_start = response:find("{")
+    local json_end = response:reverse():find("}")
+
+    if not json_start or not json_end then
+        return nil, {"No JSON object found in response"}
+    end
+
+    json_end = #response - json_end + 1
+    local json_str = response:sub(json_start, json_end)
+
+    -- Parse JSON using the json global
+    local success, parsed = pcall(function()
+        return json.decode(json_str)
+    end)
+
+    if not success then
+        return nil, {"Invalid JSON: " .. tostring(parsed)}
+    end
+
+    return parsed, {}
+end
+
+function LLMExtractor:extract(input_text)
+    local retry_count = 0
+    local last_response = nil
+    local validation_errors = {}
+
+    for attempt = 1, self.max_retries + 1 do
+        -- Build message for this attempt
+        local message
+        if attempt == 1 then
+            message = "Please extract the following information:\n\n" .. input_text
+        else
+            -- Retry with feedback
+            retry_count = retry_count + 1
+            message = self:build_retry_feedback(last_response, validation_errors)
+        end
+
+        -- Call agent
+        local agent_result = self.agent({message = message})
+        last_response = agent_result.output or ""
+
+        -- Parse and validate response
+        local parsed, parse_errors = self:parse_json(last_response)
+
+        if #parse_errors > 0 then
+            validation_errors = parse_errors
+        else
+            -- Validate extracted fields against schema
+            local result, val_errors = self:validate_fields(parsed, self.fields)
+            validation_errors = val_errors
+
+            if #validation_errors == 0 then
+                return {
+                    fields = result,
+                    retry_count = retry_count,
+                    raw_response = last_response
+                }
+            end
+        end
+    end
+
+    -- All retries exhausted
+    return {
+        fields = {},
+        error = string.format("Max retries (%d) exceeded. Validation errors: %s",
+            self.max_retries, table.concat(validation_errors, ", ")),
+        retry_count = retry_count,
+        validation_errors = validation_errors,
+        raw_response = last_response
+    }
+end
+
+function LLMExtractor:build_retry_feedback(last_response, errors)
+    local errors_str = table.concat(errors, "\n  - ")
+    local fields_list = {}
+    for name, _ in pairs(self.fields) do
+        table.insert(fields_list, '"' .. name .. '"')
+    end
+    local fields_str = table.concat(fields_list, ", ")
+
+    -- Truncate long responses
+    local response_preview = last_response
+    if #response_preview > 500 then
+        response_preview = response_preview:sub(1, 500) .. "..."
+    end
+
+    return string.format([[Your previous response was not valid JSON or had validation errors.
+
+Previous response:
+%s
+
+Errors:
+  - %s
+
+Please respond with ONLY a valid JSON object containing these fields: %s
+
+Do NOT include any explanation or text outside the JSON object.]],
+        response_preview, errors_str, fields_str)
+end
+
+function LLMExtractor:__call(text)
+    return self:extract(text)
+end
+
+-- Export LLMExtractor
+return {
+    LLMExtractor = LLMExtractor,
+}

tactus/stdlib/tac/tactus/extract.spec.tac

@@ -0,0 +1,153 @@
+--[[doc
+# Extraction Classes
+
+Proper Lua class hierarchy for structured data extraction:
+
+- **BaseExtractor**: Abstract base class with field validation
+- **LLMExtractor**: LLM-based extraction with retry logic
+
+## Usage
+
+```lua
+-- Import extraction classes
+local extract = require("tactus.extract")
+local LLMExtractor = extract.LLMExtractor
+
+-- Or load specific extractors (dependencies auto-load):
+local LLMExtractor = require("tactus.extract.llm")
+
+-- LLM Extraction
+local extractor = LLMExtractor:new {
+    fields = {name = "string", age = "number", email = "string"},
+    prompt = "Extract customer information from this text",
+    model = "openai/gpt-4o-mini"
+}
+local result = extractor:extract("John Smith is 34 years old. Contact: john@example.com")
+-- result.fields = {name = "John Smith", age = 34, email = "john@example.com"}
+```
+
+## LLMExtractor Parameters
+
+- `fields` (required): Table mapping field names to types
+- `prompt` (required): Extraction instruction
+- `model`: Model identifier (e.g., "openai/gpt-4o-mini")
+- `temperature`: LLM temperature (default: 0.3)
+- `max_retries`: Maximum retry attempts (default: 3)
+- `strict`: Require all fields (default: true)
+
+## Field Types
+
+- `string`: Text values
+- `number`: Numeric values (float)
+- `integer`: Whole numbers
+- `boolean`: true/false values
+- `list`/`array`: JSON arrays
+- `object`/`dict`: JSON objects
+]]
+
+-- Load extraction classes
+local extract = require("tactus.extract")
+local LLMExtractor = extract.LLMExtractor
+
+-- Local state for test context
+local test_state = {}
+
+-- Custom step definitions
+Step("an LLM extractor with fields (.+)", function(ctx, fields_str)
+    local fields = {}
+    -- Parse field definitions like: name:string, age:number
+    for field_def in string.gmatch(fields_str, "([^,]+)") do
+        field_def = field_def:gsub("^%s+", ""):gsub("%s+$", "")
+        local name, type_ = field_def:match("([^:]+):([^:]+)")
+        if name and type_ then
+            fields[name:gsub("^%s+", ""):gsub("%s+$", "")] = type_:gsub("^%s+", ""):gsub("%s+$", "")
+        end
+    end
+    test_state.extractor_config = {
+        fields = fields,
+        model = "openai/gpt-4o-mini"
+    }
+end)
+
+Step("extraction prompt \"(.+)\"", function(ctx, prompt)
+    test_state.extractor_config.prompt = prompt
+end)
+
+Step("I extract from \"(.+)\"", function(ctx, text)
+    if not test_state.extractor then
+        test_state.extractor = LLMExtractor:new(test_state.extractor_config)
+    end
+    test_state.result = test_state.extractor:extract(text)
+end)
+
+Step("the extracted field \"(.+)\" should be \"(.+)\"", function(ctx, field, expected)
+    assert(test_state.result, "No extraction result found")
+    assert(test_state.result.fields, "No fields in extraction result")
+    local actual = test_state.result.fields[field]
+    assert(tostring(actual) == expected,
+        "Expected field '" .. field .. "' to be '" .. expected .. "' but got '" .. tostring(actual) .. "'")
+end)
+
+Step("the extracted field \"(.+)\" should be number (.+)", function(ctx, field, expected)
+    assert(test_state.result, "No extraction result found")
+    assert(test_state.result.fields, "No fields in extraction result")
+    local actual = test_state.result.fields[field]
+    assert(type(actual) == "number",
+        "Expected field '" .. field .. "' to be a number but got " .. type(actual))
+    assert(actual == tonumber(expected),
+        "Expected field '" .. field .. "' to be " .. expected .. " but got " .. tostring(actual))
+end)
+
+Step("the extraction should succeed", function(ctx)
+    assert(test_state.result, "No extraction result found")
+    assert(not test_state.result.error,
+        "Extraction failed with error: " .. tostring(test_state.result.error))
+end)
+
+Step("the extraction should have no validation errors", function(ctx)
+    assert(test_state.result, "No extraction result found")
+    local errors = test_state.result.validation_errors or {}
+    assert(#errors == 0,
+        "Expected no validation errors but got: " .. table.concat(errors, ", "))
+end)
+
+-- BDD Specifications
+Specification([[
+Feature: Extraction Class Hierarchy
+  As a Tactus developer
+  I want to extract structured data from text
+  So that I can process unstructured information programmatically
+
+  Scenario: Extract simple contact information
+    Given an LLM extractor with fields name:string, age:number
+    And extraction prompt "Extract the person's name and age"
+    When I extract from "John Smith is 34 years old"
+    Then the extraction should succeed
+    And the extracted field "name" should be "John Smith"
+    And the extracted field "age" should be number 34
+
+  Scenario: Extract multiple string fields
+    Given an LLM extractor with fields city:string, country:string
+    And extraction prompt "Extract the city and country"
+    When I extract from "The meeting will be held in Paris, France"
+    Then the extraction should succeed
+    And the extracted field "city" should be "Paris"
+    And the extracted field "country" should be "France"
+
+  Scenario: Extract with validation
+    Given an LLM extractor with fields product:string, price:number, quantity:integer
+    And extraction prompt "Extract product details"
+    When I extract from "Order: 5 widgets at $19.99 each"
+    Then the extraction should succeed
+    And the extraction should have no validation errors
+]])
+
+-- Minimal procedure
+Procedure {
+    output = {
+        result = field.string{required = true}
+    },
+    function(input)
+        return {result = "Extraction class hierarchy specs executed"}
+    end
+}

tactus/stdlib/tac/tactus/generate/base.tac

@@ -0,0 +1,142 @@
+-- Base Generator Module
+--
+-- Provides the BaseGenerator abstract class and class helper for
+-- building custom generator implementations.
+--
+-- Inspired by DSPy's modular generation approach:
+-- - Configurable generation parameters
+-- - Optional chain-of-thought reasoning
+-- - Output format control
+-- - Retry logic for robustness
+
+-- ============================================================================
+-- Class Helper (same pattern as classify/extract modules)
+-- ============================================================================
+
+local function class(base)
+    local cls = {}
+    cls.__index = cls
+
+    if base then
+        setmetatable(cls, {__index = base})
+    end
+
+    function cls:new(config)
+        local instance = setmetatable({}, cls)
+        if instance.init then
+            instance:init(config or {})
+        end
+        return instance
+    end
+
+    return cls
+end
+
+-- ============================================================================
+-- BaseGenerator
+-- ============================================================================
+
+local BaseGenerator = class()
+
+function BaseGenerator:init(config)
+    -- Core configuration
+    self.name = config.name
+    self.model = config.model
+    self.temperature = config.temperature or 0.7
+    self.max_tokens = config.max_tokens
+    self.max_retries = config.max_retries or 2
+
+    -- Generation options (DSPy-inspired)
+    self.reasoning = config.reasoning or false  -- Chain-of-thought mode
+    self.output_format = config.output_format or "text"  -- "text", "json", "markdown"
+    self.constraints = config.constraints  -- Output constraints (optional)
+
+    -- System prompt and instructions
+    self.system_prompt = config.system_prompt
+    self.instructions = config.instructions
+end
+
+function BaseGenerator:generate(prompt)
+    error("BaseGenerator:generate() must be implemented by subclass")
+end
+
+function BaseGenerator:__call(prompt)
+    return self:generate(prompt)
+end
+
+-- ============================================================================
+-- Helper functions for subclasses
+-- ============================================================================
+
+-- Build system prompt
+-- Note: Reasoning is handled by DSPy's ChainOfThought module, not manual prompts
+function BaseGenerator:build_system_prompt()
+    local parts = {}
+
+    -- Base system prompt
+    if self.system_prompt then
+        table.insert(parts, self.system_prompt)
+    else
+        table.insert(parts, "You are a helpful assistant.")
+    end
+
+    -- Note: reasoning is NOT added to system prompt here
+    -- When reasoning=true, we use DSPy's ChainOfThought module which handles
+    -- reasoning automatically without modifying the prompt
+
+    -- Add output format instructions
+    if self.output_format == "json" then
+        table.insert(parts, "Respond with valid JSON only. No markdown formatting or code blocks.")
+    elseif self.output_format == "markdown" then
+        table.insert(parts, "Format your response using Markdown.")
+    end
+
+    -- Add custom instructions
+    if self.instructions then
+        table.insert(parts, self.instructions)
+    end
+
+    -- Add constraints
+    if self.constraints then
+        if type(self.constraints) == "table" then
+            table.insert(parts, "Constraints: " .. table.concat(self.constraints, ", "))
+        else
+            table.insert(parts, "Constraints: " .. self.constraints)
+        end
+    end
+
+    return table.concat(parts, "\n\n")
+end
+
+-- Parse response to extract reasoning and final response
+function BaseGenerator:parse_reasoning_response(response)
+    if not self.reasoning then
+        return {
+            response = response,
+            reasoning = nil
+        }
+    end
+
+    -- Try to extract REASONING and RESPONSE sections
+    local reasoning = response:match("REASONING:%s*(.-)%s*RESPONSE:")
+    local final_response = response:match("RESPONSE:%s*(.*)$")
+
+    if reasoning and final_response then
+        return {
+            response = final_response:gsub("^%s+", ""):gsub("%s+$", ""),
+            reasoning = reasoning:gsub("^%s+", ""):gsub("%s+$", "")
+        }
+    end
+
+    -- If pattern doesn't match, return whole response
+    return {
+        response = response,
+        reasoning = nil
+    }
+end
+
+-- Export BaseGenerator and class helper
+return {
+    BaseGenerator = BaseGenerator,
+    class = class,
+}