amsdal_ml 0.1.2__tar.gz → 0.1.4__tar.gz

{amsdal_ml-0.1.2 → amsdal_ml-0.1.4}/.github/workflows/ci.yml

@@ -11,10 +11,10 @@ jobs:
       - name: Reset permissions
         run: |
           sudo chown -R $(id -u):$(id -g) .
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Python install
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.11"
       - name: License check
@@ -36,19 +36,19 @@ jobs:
       DEPS: yes
       _TYPER_FORCE_DISABLE_TERMINAL: true
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: szenius/set-timezone@v2.0
         with:
           timezoneLinux: "EEST"
 
       - name: Set up python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Hatch and UV setup
         run: |
-          pip install --upgrade uv hatch
+          pip install --upgrade uv hatch==1.14.2
           hatch env prune
           hatch env create
           hatch run sync

{amsdal_ml-0.1.2 → amsdal_ml-0.1.4}/.github/workflows/release.yml

@@ -10,10 +10,10 @@ jobs:
     name: License check
     runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Python install
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.11"
       - name: License check
@@ -34,17 +34,17 @@ jobs:
       DEPS: yes
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set up python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install hatch
         run: |
           python -m pip install --upgrade setuptools
-          python -m pip install --upgrade hatch
+          python -m pip install hatch==1.14.2
 
       - name: Build
         run: hatch build
@@ -65,7 +65,7 @@ jobs:
         python-version: [ "3.10" ]
     steps:
       - name: Set up python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -83,7 +83,7 @@ jobs:
       - name: Install hatch
         run: |
           python -m pip install --upgrade setuptools
-          python -m pip install --upgrade hatch
+          python -m pip install hatch==1.14.2
 
       - name: Publish
         run: |

{amsdal_ml-0.1.2 → amsdal_ml-0.1.4}/.github/workflows/tag_check.yml

@@ -13,7 +13,7 @@ jobs:
     if: github.event.pull_request.merged == true && github.event.pull_request.base.ref == 'main' && startsWith(github.event.pull_request.head.ref, 'release/')
     runs-on: self-hosted
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
         with:
           token: ${{ secrets.ACCESS_KEY }}
 

amsdal_ml-0.1.4/CLAUDE.md

@@ -0,0 +1,171 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+amsdal-ml is a machine learning plugin for the AMSDAL Framework that provides embeddings, vector search, and AI-driven features. It supports both synchronous and asynchronous modes, with primary focus on async operations using OpenAI models.
+
+## Development Commands
+
+### Environment Setup
+```bash
+# Install dependencies using hatch/uv
+pip install --upgrade uv hatch==1.14.2
+hatch env create
+hatch run sync
+```
+
+### Testing
+```bash
+# Run all tests with coverage
+hatch run cov
+
+# Run specific test file
+hatch run test tests/test_openai_model.py
+
+# Run tests with pytest directly (after env setup)
+pytest tests/
+pytest tests/agents_tests/  # Run agent-specific tests
+```
+
+### Code Quality
+```bash
+# Run all checks (style + typing)
+hatch run all
+
+# Style checks only
+hatch run style
+
+# Format code (fix style issues)
+hatch run fmt
+
+# Type checking
+hatch run typing
+```
+
+### Dependency Management
+```bash
+# Sync dependencies
+hatch run sync
+
+# Update lock file
+hatch run lock
+
+# Upgrade all dependencies
+hatch run lock-upgrade
+```
+
+### AMSDAL CLI Commands
+```bash
+# Generate new model
+amsdal generate model ModelName --format py
+
+# Generate property for model
+amsdal generate property --model ModelName property_name
+
+# Generate transaction
+amsdal generate transaction TransactionName
+
+# Generate hook
+amsdal generate hook --model ModelName on_create
+```
+
+## Architecture
+
+### Core Components
+
+**ML Models** (`amsdal_ml/ml_models/`)
+- Abstract base class `MLModel` defines the interface for all ML models
+- Supports both sync/async invoke and streaming methods
+- Primary implementation uses OpenAI API
+- All models must implement `setup()`, `teardown()`, `invoke()`, `ainvoke()`, `stream()`, and `astream()`
+- Custom error hierarchy: `ModelError`, `ModelConnectionError`, `ModelRateLimitError`, `ModelAPIError`
+
+**ML Ingesting** (`amsdal_ml/ml_ingesting/`)
+- `MLIngesting` abstract base handles text generation and embedding creation from data
+- Creates `EmbeddingData` records that link embeddings to source objects
+- Supports chunk-based processing with configurable depth and token limits
+- Both sync/async methods for text generation and embedding
+
+**ML Retrievers** (`amsdal_ml/ml_retrievers/`)
+- `MLRetriever` provides semantic search via similarity_search/asimilarity_search
+- Returns `RetrievalChunk` objects with object metadata, chunk text, distance, and tags
+- Supports filtering by include/exclude tags
+- Configurable k parameter for number of results
+
+**Agents** (`amsdal_ml/agents/`)
+- Abstract `Agent` base class for Q&A and task-oriented agents
+- Async-first design (sync methods raise NotImplementedError)
+- Returns `AgentOutput` with answer, used_tools, and citations
+- Supports streaming responses via `astream()`
+- File attachments supported through `FileAttachment` interface
+
+**MCP Integration**
+- **Server** (`amsdal_ml/mcp_server/`): Exposes retriever search as MCP tool via stdio
+- **Client** (`amsdal_ml/mcp_client/`): Supports both stdio and HTTP transports for calling MCP tools
+- Server accepts base64-encoded AMSDAL config for initialization
+
+**File I/O** (`amsdal_ml/fileio/`)
+- `BaseFileLoader` abstract class for uploading files to ML providers
+- `FileAttachment` represents processed attachments (types: PLAIN_TEXT, FILE_ID)
+- `FileItem` helper for creating attachments from paths, bytes, or strings
+
+### Data Models
+
+**EmbeddingModel** (`amsdal_ml/models/embedding_model.py`)
+- Core model storing embeddings in database
+- Links to source object via `data_object_class` and `data_object_id`
+- Stores 1536-dimensional vectors (OpenAI text-embedding-3-small default)
+- Includes chunk_index, raw_text, tags, and ml_metadata fields
+
+### Configuration
+
+**MLConfig** (`amsdal_ml/ml_config.py`)
+- Loaded from `.env` file using pydantic-settings
+- Key settings:
+  - `ml_model_class`: Path to ML model implementation
+  - `ml_retriever_class`: Path to retriever implementation
+  - `ml_ingesting_class`: Path to ingesting implementation
+  - `llm_model_name`: Default 'gpt-4o'
+  - `embed_model_name`: Default 'text-embedding-3-small'
+  - `embed_max_depth`, `embed_max_chunks`, `embed_max_tokens_per_chunk`: Chunking parameters
+  - `retriever_default_k`: Number of results for similarity search
+  - `openai_api_key`, `claude_api_key`: API credentials
+  - `embedding_targets`: List of models to embed
+
+**Database Config** (`config.yml`)
+- Defines AMSDAL connections (sqlite_history, sqlite_state, lock)
+- Resources config maps repository and lakehouse to connections
+- Set `async_mode: true` for async operations
+
+## Code Style
+
+- Python 3.11+ required
+- Uses Ruff for linting and formatting with 120-char line length
+- Single quotes enforced (`quote-style = "single"`)
+- Import ordering: force-single-line with order-by-type
+- Type checking via mypy with strict settings (disallow_any_generics, check_untyped_defs)
+- Excludes migrations directory from linting
+
+## Testing
+
+- Uses pytest with pytest-asyncio for async tests
+- Test fixtures in `tests/conftest.py` provide mocked OpenAI clients
+- `OPENAI_API_KEY` set to dummy value in tests via fixture
+- Coverage tracking with coverage.py
+
+## CI/CD
+
+The project uses self-hosted runners with two jobs:
+1. **license-check**: Validates third-party licenses using `license_check.py`
+2. **test-lint**: Runs on Python 3.11 and 3.12, executes `hatch run all` (style+typing) and `hatch run cov`
+
+## Key Patterns
+
+1. **Async-First**: Most components prioritize async methods; sync methods often raise NotImplementedError
+2. **Abstract Base Classes**: Heavy use of ABCs to define interfaces for models, retrievers, ingesters, and agents
+3. **Configuration via Pydantic**: Settings loaded from environment with type validation
+4. **AMSDAL Integration**: Uses AMSDAL's model system, manager, and connection framework
+5. **Chunking Strategy**: Text split into chunks with metadata preservation for better embedding quality
+6. **Tag-Based Filtering**: Embeddings tagged for fine-grained retrieval control

amsdal_ml-0.1.4/PKG-INFO

@@ -0,0 +1,235 @@
+Metadata-Version: 2.4
+Name: amsdal_ml
+Version: 0.1.4
+Summary: amsdal_ml plugin for AMSDAL Framework
+Requires-Python: >=3.11
+Requires-Dist: aiohttp==3.12.15
+Requires-Dist: amsdal-cli>=0.5.7
+Requires-Dist: amsdal-data>=0.5.9
+Requires-Dist: amsdal-models>=0.5.9
+Requires-Dist: amsdal-utils>=0.5.4
+Requires-Dist: amsdal>=0.5.6
+Requires-Dist: mcp>=0.1
+Requires-Dist: openai==1.100.2
+Requires-Dist: pydantic-settings==2.10.1
+Requires-Dist: pydantic==2.11.7
+Description-Content-Type: text/markdown
+
+# AMSDAL ML
+
+[![CI](https://github.com/amsdal/amsdal_ml/actions/workflows/ci.yml/badge.svg)](https://github.com/amsdal/amsdal_ml/actions/workflows/ci.yml)
+[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/)
+
+Machine learning plugin for the AMSDAL Framework, providing embeddings, vector search, semantic retrieval, and AI agents with support for OpenAI models.
+
+## Features
+
+- **Vector Embeddings**: Generate and store embeddings for any AMSDAL model with automatic chunking
+- **Semantic Search**: Query your data using natural language with tag-based filtering
+- **AI Agents**: Build Q&A systems with streaming support and citation tracking
+- **Async-First**: Optimized for high-performance async operations
+- **MCP Integration**: Expose and consume tools via Model Context Protocol (stdio/HTTP)
+- **File Attachments**: Process and embed documents with built-in loaders
+- **Extensible**: Abstract base classes for custom models, retrievers, and ingesters
+
+## Installation
+
+```bash
+pip install amsdal-ml
+```
+
+### Requirements
+
+- Python 3.11 or higher
+- AMSDAL Framework 0.5.6+
+- OpenAI API key (for default implementations)
+
+## Quick Start
+
+### 1. Configuration
+
+Create a `.env` file in your project root:
+
+```env
+OPENAI_API_KEY=sk-your-api-key-here
+async_mode=true
+ml_model_class=amsdal_ml.ml_models.openai_model.OpenAIModel
+ml_retriever_class=amsdal_ml.ml_retrievers.openai_retriever.OpenAIRetriever
+ml_ingesting_class=amsdal_ml.ml_ingesting.openai_ingesting.OpenAIIngesting
+```
+
+Create a `config.yml` for AMSDAL connections:
+
+```yaml
+application_name: my-ml-app
+async_mode: true
+connections:
+  - name: sqlite_state
+    backend: sqlite-state-async
+    credentials:
+      - db_path: ./warehouse/state.sqlite3
+      - check_same_thread: false
+  - name: lock
+    backend: amsdal_data.lock.implementations.thread_lock.ThreadLock
+resources_config:
+  repository:
+    default: sqlite_state
+  lock: lock
+```
+
+### 2. Generate Embeddings
+
+```python
+from amsdal_ml.ml_ingesting.openai_ingesting import OpenAIIngesting
+from amsdal_ml.ml_config import ml_config
+
+# Initialize ingesting
+ingester = OpenAIIngesting(
+    model=MyModel,
+    embedding_field='embedding',
+)
+
+# Generate embeddings for an instance
+instance = MyModel(content='Your text here')
+embeddings = await ingester.agenerate_embeddings(instance)
+await ingester.asave(embeddings, instance)
+```
+
+### 3. Semantic Search
+
+```python
+from amsdal_ml.ml_retrievers.openai_retriever import OpenAIRetriever
+
+retriever = OpenAIRetriever()
+
+# Search for relevant content
+results = await retriever.asimilarity_search(
+    query='What is machine learning?',
+    k=5,
+    include_tags=['documentation']
+)
+
+for chunk in results:
+    print(f'{chunk.object_class}:{chunk.object_id} - {chunk.raw_text}')
+```
+
+### 4. Build an AI Agent
+
+```python
+from amsdal_ml.agents.default_qa_agent import DefaultQAAgent
+
+agent = DefaultQAAgent()
+
+# Ask questions
+output = await agent.arun('Explain vector embeddings')
+print(output.answer)
+print(f'Used tools: {output.used_tools}')
+
+# Stream responses
+async for chunk in agent.astream('What is semantic search?'):
+    print(chunk, end='', flush=True)
+```
+
+## Architecture
+
+### Core Components
+
+- **`MLModel`**: Abstract interface for LLM inference (invoke, stream, with attachments)
+- **`MLIngesting`**: Generate text and embeddings from data objects with chunking
+- **`MLRetriever`**: Semantic similarity search with tag-based filtering
+- **`Agent`**: Q&A and task-oriented agents with streaming and citations
+- **`EmbeddingModel`**: Database model storing 1536-dimensional vectors linked to source objects
+- **MCP Server/Client**: Expose retrievers as tools or consume external MCP services
+
+### Configuration
+
+All settings are managed via `MLConfig` in `.env`:
+
+```env
+# Model Configuration
+llm_model_name=gpt-4o
+llm_temperature=0.0
+embed_model_name=text-embedding-3-small
+
+# Chunking Parameters
+embed_max_depth=2
+embed_max_chunks=10
+embed_max_tokens_per_chunk=800
+
+# Retrieval Settings
+retriever_default_k=8
+```
+
+## Development
+
+### Setup
+
+```bash
+# Install dependencies
+pip install --upgrade uv hatch==1.14.2
+hatch env create
+hatch run sync
+```
+
+### Testing
+
+```bash
+# Run all tests with coverage
+hatch run cov
+
+# Run specific tests
+hatch run test tests/test_openai_model.py
+
+# Watch mode
+pytest tests/ -v
+```
+
+### Code Quality
+
+```bash
+# Run all checks (style + typing)
+hatch run all
+
+# Format code
+hatch run fmt
+
+# Type checking
+hatch run typing
+```
+
+### AMSDAL CLI
+
+```bash
+# Generate a new model
+amsdal generate model MyModel --format py
+
+# Generate property
+amsdal generate property --model MyModel embedding_field
+
+# Generate transaction
+amsdal generate transaction ProcessEmbeddings
+
+# Generate hook
+amsdal generate hook --model MyModel on_create
+```
+
+## MCP Server
+
+Run the retriever as an MCP server for integration with Claude Desktop or other MCP clients:
+
+```bash
+python -m amsdal_ml.mcp_server.server_retriever_stdio \
+  --amsdal-config "$(echo '{"async_mode": true, ...}' | base64)"
+```
+
+The server exposes a `search` tool for semantic search in your knowledge base.
+
+## License
+
+See `amsdal_ml/Third-Party Materials - AMSDAL Dependencies - License Notices.md` for dependency licenses.
+
+## Links
+
+- [AMSDAL Framework](https://github.com/amsdal/amsdal)
+- [Documentation](https://docs.amsdal.com)
+- [Issue Tracker](https://github.com/amsdal/amsdal_ml/issues)