openadapt-ml 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- openadapt_ml-0.1.0/.env.example +132 -0
- openadapt_ml-0.1.0/.github/workflows/publish.yml +30 -0
- openadapt_ml-0.1.0/.gitignore +41 -0
- openadapt_ml-0.1.0/.gitmodules +3 -0
- openadapt_ml-0.1.0/.python-version +1 -0
- openadapt_ml-0.1.0/CLAUDE.md +620 -0
- openadapt_ml-0.1.0/LICENSE +21 -0
- openadapt_ml-0.1.0/PKG-INFO +818 -0
- openadapt_ml-0.1.0/README.md +771 -0
- openadapt_ml-0.1.0/configs/qwen2_5vl_synthetic.yaml +25 -0
- openadapt_ml-0.1.0/configs/qwen3vl_capture.yaml +31 -0
- openadapt_ml-0.1.0/configs/qwen3vl_capture_4bit.yaml +30 -0
- openadapt_ml-0.1.0/configs/qwen3vl_capture_batched.yaml +32 -0
- openadapt_ml-0.1.0/configs/qwen3vl_synthetic.yaml +29 -0
- openadapt_ml-0.1.0/configs/qwen3vl_synthetic_coord_v2.yaml +30 -0
- openadapt_ml-0.1.0/configs/qwen3vl_synthetic_dev.yaml +30 -0
- openadapt_ml-0.1.0/configs/qwen3vl_synthetic_registration_som.yaml +31 -0
- openadapt_ml-0.1.0/configs/qwen3vl_synthetic_som.yaml +30 -0
- openadapt_ml-0.1.0/docs/NEXT_STEPS_GROUNDING_ARCHITECTURE.md +336 -0
- openadapt_ml-0.1.0/docs/PRIVACY_IMPLEMENTATION_PLAN.md +293 -0
- openadapt_ml-0.1.0/docs/RECORD_IMPLEMENTATION_PLAN.md +376 -0
- openadapt_ml-0.1.0/docs/auto_shutoff_design.md +129 -0
- openadapt_ml-0.1.0/docs/azure_acr_authentication.md +159 -0
- openadapt_ml-0.1.0/docs/azure_waa_setup.md +308 -0
- openadapt_ml-0.1.0/docs/batching_and_schedulers.md +351 -0
- openadapt_ml-0.1.0/docs/benchmark_integration_plan.md +677 -0
- openadapt_ml-0.1.0/docs/benchmark_next_steps.md +105 -0
- openadapt_ml-0.1.0/docs/benchmark_viewer_integration.md +200 -0
- openadapt_ml-0.1.0/docs/benchmark_viewer_phase2.md +343 -0
- openadapt_ml-0.1.0/docs/cloud_gpu_training.md +124 -0
- openadapt_ml-0.1.0/docs/dashboard_architecture.md +77 -0
- openadapt_ml-0.1.0/docs/design.md +569 -0
- openadapt_ml-0.1.0/docs/early_termination.md +230 -0
- openadapt_ml-0.1.0/docs/eval_json_schema.md +361 -0
- openadapt_ml-0.1.0/docs/gemini_grounding.md +313 -0
- openadapt_ml-0.1.0/docs/gui_actor_integration.md +131 -0
- openadapt_ml-0.1.0/docs/images/dashboard/training_bottom.png +0 -0
- openadapt_ml-0.1.0/docs/images/dashboard/training_top.png +0 -0
- openadapt_ml-0.1.0/docs/images/dashboard/viewer_bottom.png +0 -0
- openadapt_ml-0.1.0/docs/images/dashboard/viewer_top.png +0 -0
- openadapt_ml-0.1.0/docs/images/grounding_demo.png +0 -0
- openadapt_ml-0.1.0/docs/images/grounding_demo_full.png +0 -0
- openadapt_ml-0.1.0/docs/images/training-dashboard.png +0 -0
- openadapt_ml-0.1.0/docs/images/viewer-comparison.png +0 -0
- openadapt_ml-0.1.0/docs/live_inference_design.md +340 -0
- openadapt_ml-0.1.0/docs/opencua_integration.md +581 -0
- openadapt_ml-0.1.0/docs/output_artifacts_and_media.md +123 -0
- openadapt_ml-0.1.0/docs/prediction_loading_architecture.md +215 -0
- openadapt_ml-0.1.0/docs/qwen_login_experiment.md +729 -0
- openadapt_ml-0.1.0/docs/roadmap.md +395 -0
- openadapt_ml-0.1.0/docs/set_of_marks_implementation.md +324 -0
- openadapt_ml-0.1.0/docs/som_implementation_verification.md +368 -0
- openadapt_ml-0.1.0/docs/state_and_next_steps_qwen_login.md +604 -0
- openadapt_ml-0.1.0/docs/stub_training_adapter.md +132 -0
- openadapt_ml-0.1.0/docs/synthetic_login_jitter_and_ablation.md +162 -0
- openadapt_ml-0.1.0/docs/training_feedback_ux.md +225 -0
- openadapt_ml-0.1.0/docs/unified_compute_architecture.md +225 -0
- openadapt_ml-0.1.0/docs/viewer_eval_integration.md +169 -0
- openadapt_ml-0.1.0/docs/viewer_layout_redesign.md +137 -0
- openadapt_ml-0.1.0/docs/vision.md +71 -0
- openadapt_ml-0.1.0/docs/wandb_integration.md +327 -0
- openadapt_ml-0.1.0/examples/README.md +64 -0
- openadapt_ml-0.1.0/examples/sample_data.json +97 -0
- openadapt_ml-0.1.0/examples/test_gemini_grounding.py +113 -0
- openadapt_ml-0.1.0/examples/train_from_json.py +153 -0
- openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/media/qwen3_2b_login_demo.gif +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/media/qwen3_2b_login_demo_session_0001.gif +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/plots/base_vs_ft.png +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/plots/qwen3_2b_base_vs_ft_hardened_v2.png +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/plots/qwen_vs_apis.png +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/8b_hero/plots/qwen3_8b_base_vs_ft_hardened_v2.png +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/SOM_INVESTIGATION_REPORT.md +458 -0
- openadapt_ml-0.1.0/experiments/qwen_login/comprehensive_comparison.png +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/login_demo.gif +0 -0
- openadapt_ml-0.1.0/experiments/qwen_login/registration_demo.gif +0 -0
- openadapt_ml-0.1.0/openadapt_ml/__init__.py +0 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/__init__.py +125 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/agent.py +825 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/azure.py +761 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/base.py +366 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/cli.py +884 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/data_collection.py +432 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/runner.py +381 -0
- openadapt_ml-0.1.0/openadapt_ml/benchmarks/waa.py +704 -0
- openadapt_ml-0.1.0/openadapt_ml/cloud/__init__.py +5 -0
- openadapt_ml-0.1.0/openadapt_ml/cloud/azure_inference.py +441 -0
- openadapt_ml-0.1.0/openadapt_ml/cloud/lambda_labs.py +2445 -0
- openadapt_ml-0.1.0/openadapt_ml/cloud/local.py +790 -0
- openadapt_ml-0.1.0/openadapt_ml/config.py +56 -0
- openadapt_ml-0.1.0/openadapt_ml/datasets/__init__.py +0 -0
- openadapt_ml-0.1.0/openadapt_ml/datasets/next_action.py +507 -0
- openadapt_ml-0.1.0/openadapt_ml/evals/__init__.py +23 -0
- openadapt_ml-0.1.0/openadapt_ml/evals/grounding.py +241 -0
- openadapt_ml-0.1.0/openadapt_ml/evals/plot_eval_metrics.py +174 -0
- openadapt_ml-0.1.0/openadapt_ml/evals/trajectory_matching.py +486 -0
- openadapt_ml-0.1.0/openadapt_ml/grounding/__init__.py +45 -0
- openadapt_ml-0.1.0/openadapt_ml/grounding/base.py +236 -0
- openadapt_ml-0.1.0/openadapt_ml/grounding/detector.py +570 -0
- openadapt_ml-0.1.0/openadapt_ml/ingest/__init__.py +43 -0
- openadapt_ml-0.1.0/openadapt_ml/ingest/capture.py +312 -0
- openadapt_ml-0.1.0/openadapt_ml/ingest/loader.py +232 -0
- openadapt_ml-0.1.0/openadapt_ml/ingest/synthetic.py +1102 -0
- openadapt_ml-0.1.0/openadapt_ml/models/__init__.py +0 -0
- openadapt_ml-0.1.0/openadapt_ml/models/api_adapter.py +171 -0
- openadapt_ml-0.1.0/openadapt_ml/models/base_adapter.py +59 -0
- openadapt_ml-0.1.0/openadapt_ml/models/dummy_adapter.py +42 -0
- openadapt_ml-0.1.0/openadapt_ml/models/qwen_vl.py +426 -0
- openadapt_ml-0.1.0/openadapt_ml/runtime/__init__.py +0 -0
- openadapt_ml-0.1.0/openadapt_ml/runtime/policy.py +182 -0
- openadapt_ml-0.1.0/openadapt_ml/schemas/__init__.py +53 -0
- openadapt_ml-0.1.0/openadapt_ml/schemas/sessions.py +122 -0
- openadapt_ml-0.1.0/openadapt_ml/schemas/validation.py +252 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/__init__.py +0 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/compare.py +1490 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/demo_policy.py +62 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/eval_policy.py +287 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/make_gif.py +153 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/prepare_synthetic.py +43 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/run_qwen_login_benchmark.py +192 -0
- openadapt_ml-0.1.0/openadapt_ml/scripts/train.py +174 -0
- openadapt_ml-0.1.0/openadapt_ml/training/__init__.py +0 -0
- openadapt_ml-0.1.0/openadapt_ml/training/benchmark_viewer.py +1538 -0
- openadapt_ml-0.1.0/openadapt_ml/training/shared_ui.py +157 -0
- openadapt_ml-0.1.0/openadapt_ml/training/stub_provider.py +276 -0
- openadapt_ml-0.1.0/openadapt_ml/training/trainer.py +2446 -0
- openadapt_ml-0.1.0/openadapt_ml/training/viewer.py +2970 -0
- openadapt_ml-0.1.0/pyproject.toml +68 -0
- openadapt_ml-0.1.0/scripts/fix_acr_auth.py +212 -0
- openadapt_ml-0.1.0/scripts/setup_azure.py +1158 -0
- openadapt_ml-0.1.0/tests/__init__.py +0 -0
- openadapt_ml-0.1.0/tests/benchmarks/__init__.py +0 -0
- openadapt_ml-0.1.0/tests/benchmarks/test_api_agent.py +370 -0
- openadapt_ml-0.1.0/tests/benchmarks/test_waa.py +198 -0
- openadapt_ml-0.1.0/tests/test_action_parsing.py +138 -0
- openadapt_ml-0.1.0/tests/test_api_adapter.py +60 -0
- openadapt_ml-0.1.0/tests/test_local_cli.py +257 -0
- openadapt_ml-0.1.0/tests/test_training_dummy.py +26 -0
- openadapt_ml-0.1.0/uv.lock +3216 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# API Keys for VLM backends
|
|
2
|
+
ANTHROPIC_API_KEY=your-anthropic-api-key-here
|
|
3
|
+
OPENAI_API_KEY=your-openai-api-key-here
|
|
4
|
+
|
|
5
|
+
# =============================================================================
|
|
6
|
+
# Lambda Labs (Cloud GPU for fast training)
|
|
7
|
+
# =============================================================================
|
|
8
|
+
# Get your API key at: https://cloud.lambdalabs.com/api-keys
|
|
9
|
+
#
|
|
10
|
+
# Usage:
|
|
11
|
+
# python -m openadapt_ml.cloud.lambda_labs list # See available GPUs
|
|
12
|
+
# python -m openadapt_ml.cloud.lambda_labs launch # Launch A100 instance
|
|
13
|
+
# python -m openadapt_ml.cloud.lambda_labs terminate <id> # Stop billing!
|
|
14
|
+
#
|
|
15
|
+
LAMBDA_API_KEY=your-lambda-api-key-here
|
|
16
|
+
|
|
17
|
+
# Google Gemini API (for GeminiGrounder element detection)
|
|
18
|
+
# Get your key via one of these methods:
|
|
19
|
+
#
|
|
20
|
+
# Option 1: Google AI Studio (easiest)
|
|
21
|
+
# 1. Go to https://aistudio.google.com/apikey
|
|
22
|
+
# 2. Click "Create API Key"
|
|
23
|
+
# 3. Select or create a Google Cloud project
|
|
24
|
+
# 4. Copy the key
|
|
25
|
+
#
|
|
26
|
+
# Option 2: Google Cloud Console
|
|
27
|
+
# 1. Go to https://console.cloud.google.com/apis/credentials
|
|
28
|
+
# 2. Select your project
|
|
29
|
+
# 3. Click "Create Credentials" → "API Key"
|
|
30
|
+
# 4. Enable "Generative Language API" at:
|
|
31
|
+
# https://console.cloud.google.com/apis/library/generativelanguage.googleapis.com
|
|
32
|
+
#
|
|
33
|
+
GOOGLE_API_KEY=your-google-api-key-here
|
|
34
|
+
|
|
35
|
+
# =============================================================================
|
|
36
|
+
# Azure Credentials (for WAA benchmark evaluation on Azure VMs)
|
|
37
|
+
# =============================================================================
|
|
38
|
+
#
|
|
39
|
+
# EASY SETUP: Run the automated setup script:
|
|
40
|
+
# python scripts/setup_azure.py
|
|
41
|
+
#
|
|
42
|
+
# This script will:
|
|
43
|
+
# - Check Azure CLI installation
|
|
44
|
+
# - Log you in (opens browser)
|
|
45
|
+
# - Create service principal, resource group, and ML workspace
|
|
46
|
+
# - Write credentials to this file
|
|
47
|
+
#
|
|
48
|
+
# MANUAL SETUP: Follow the steps below if you prefer manual control.
|
|
49
|
+
#
|
|
50
|
+
# Azure authentication uses Service Principal credentials. To create them:
|
|
51
|
+
#
|
|
52
|
+
# Step 1: Install Azure CLI
|
|
53
|
+
# macOS: brew install azure-cli
|
|
54
|
+
# Windows: winget install Microsoft.AzureCLI
|
|
55
|
+
# Linux: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
|
|
56
|
+
#
|
|
57
|
+
# Step 2: Login to Azure
|
|
58
|
+
# az login
|
|
59
|
+
#
|
|
60
|
+
# Step 3: Get your subscription ID
|
|
61
|
+
# az account show --query id -o tsv
|
|
62
|
+
#
|
|
63
|
+
# Step 4: Create a Service Principal (replace <subscription-id>)
|
|
64
|
+
# az ad sp create-for-rbac \
|
|
65
|
+
# --name "openadapt-ml-waa" \
|
|
66
|
+
# --role "Contributor" \
|
|
67
|
+
# --scopes "/subscriptions/<subscription-id>" \
|
|
68
|
+
# --sdk-auth
|
|
69
|
+
#
|
|
70
|
+
# This outputs JSON with clientId, clientSecret, tenantId - copy these below.
|
|
71
|
+
#
|
|
72
|
+
# Step 5: Create Azure ML Workspace (if you don't have one)
|
|
73
|
+
# az ml workspace create \
|
|
74
|
+
# --name agents_ml \
|
|
75
|
+
# --resource-group agents \
|
|
76
|
+
# --location eastus
|
|
77
|
+
#
|
|
78
|
+
# Step 6: Request vCPU quota increase (if needed)
|
|
79
|
+
# Go to: https://portal.azure.com/#view/Microsoft_Azure_Capacity/QuotaMenuBlade
|
|
80
|
+
# Select your subscription → Compute → Standard Dv3 Family
|
|
81
|
+
# Request increase to at least 320 vCPUs (for 40 workers × 8 vCPUs each)
|
|
82
|
+
#
|
|
83
|
+
# Alternative: Use Azure CLI login (no service principal needed)
|
|
84
|
+
# If AZURE_CLIENT_ID is not set, DefaultAzureCredential will try:
|
|
85
|
+
# 1. Environment variables (service principal)
|
|
86
|
+
# 2. Azure CLI credentials (az login)
|
|
87
|
+
# 3. Managed Identity (when running on Azure)
|
|
88
|
+
#
|
|
89
|
+
# Service Principal credentials (from Step 4 output)
|
|
90
|
+
AZURE_CLIENT_ID=your-client-id-here
|
|
91
|
+
AZURE_CLIENT_SECRET=your-client-secret-here
|
|
92
|
+
AZURE_TENANT_ID=your-tenant-id-here
|
|
93
|
+
|
|
94
|
+
# Azure ML Workspace config
|
|
95
|
+
AZURE_SUBSCRIPTION_ID=your-subscription-id-here
|
|
96
|
+
AZURE_ML_RESOURCE_GROUP=agents
|
|
97
|
+
AZURE_ML_WORKSPACE_NAME=agents_ml
|
|
98
|
+
|
|
99
|
+
# Optional: Override default VM settings
|
|
100
|
+
# AZURE_VM_SIZE=Standard_D8_v3
|
|
101
|
+
# AZURE_DOCKER_IMAGE=ghcr.io/microsoft/windowsagentarena:latest
|
|
102
|
+
|
|
103
|
+
# =============================================================================
|
|
104
|
+
# Azure Storage for Async Inference Queue (Phase 2)
|
|
105
|
+
# =============================================================================
|
|
106
|
+
#
|
|
107
|
+
# These settings are auto-configured by setup_azure.py (steps 11-12).
|
|
108
|
+
# They enable live inference during training:
|
|
109
|
+
# - Training uploads checkpoints to blob storage
|
|
110
|
+
# - Inference worker polls queue for new checkpoints
|
|
111
|
+
# - Results uploaded to blob for dashboard to display
|
|
112
|
+
#
|
|
113
|
+
# Setup: Run 'python scripts/setup_azure.py' to create storage account and queue
|
|
114
|
+
#
|
|
115
|
+
# Manual setup (advanced):
|
|
116
|
+
# 1. Create storage account:
|
|
117
|
+
# az storage account create --name openadaptmlstorage \
|
|
118
|
+
# --resource-group agents --location eastus --sku Standard_LRS
|
|
119
|
+
#
|
|
120
|
+
# 2. Get connection string:
|
|
121
|
+
# az storage account show-connection-string \
|
|
122
|
+
# --name openadaptmlstorage -o tsv
|
|
123
|
+
#
|
|
124
|
+
# 3. Create queue and containers:
|
|
125
|
+
# az storage queue create --name inference-jobs --connection-string <conn-str>
|
|
126
|
+
# az storage container create --name checkpoints --connection-string <conn-str>
|
|
127
|
+
# az storage container create --name comparisons --connection-string <conn-str>
|
|
128
|
+
#
|
|
129
|
+
AZURE_STORAGE_CONNECTION_STRING=your-storage-connection-string-here
|
|
130
|
+
AZURE_INFERENCE_QUEUE_NAME=inference-jobs
|
|
131
|
+
AZURE_CHECKPOINTS_CONTAINER=checkpoints
|
|
132
|
+
AZURE_COMPARISONS_CONTAINER=comparisons
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write # Required for trusted publishing
|
|
13
|
+
contents: read # Required for checkout
|
|
14
|
+
|
|
15
|
+
steps:
|
|
16
|
+
- uses: actions/checkout@v4
|
|
17
|
+
|
|
18
|
+
- name: Install uv
|
|
19
|
+
uses: astral-sh/setup-uv@v4
|
|
20
|
+
with:
|
|
21
|
+
version: "latest"
|
|
22
|
+
|
|
23
|
+
- name: Set up Python
|
|
24
|
+
run: uv python install 3.12
|
|
25
|
+
|
|
26
|
+
- name: Build package
|
|
27
|
+
run: uv build
|
|
28
|
+
|
|
29
|
+
- name: Publish to PyPI
|
|
30
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Python-generated files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[oc]
|
|
4
|
+
build/
|
|
5
|
+
dist/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info
|
|
8
|
+
|
|
9
|
+
# Virtual environments
|
|
10
|
+
.venv
|
|
11
|
+
local_context_openadapt_ml_internal.md
|
|
12
|
+
|
|
13
|
+
# Environment variables
|
|
14
|
+
.env
|
|
15
|
+
|
|
16
|
+
# Ephemeral synthetic assets (frames, debug sessions, etc.)
|
|
17
|
+
synthetic/
|
|
18
|
+
synthetic_train/
|
|
19
|
+
synthetic_train_dev/
|
|
20
|
+
|
|
21
|
+
# Local OS cruft
|
|
22
|
+
.DS_Store
|
|
23
|
+
|
|
24
|
+
# Editor files
|
|
25
|
+
*.swp
|
|
26
|
+
*.swo
|
|
27
|
+
*.swn
|
|
28
|
+
*~
|
|
29
|
+
.*.sw?
|
|
30
|
+
|
|
31
|
+
# Ephemeral training/eval artifacts
|
|
32
|
+
logs/
|
|
33
|
+
checkpoints/
|
|
34
|
+
checkpoints_*/
|
|
35
|
+
/plots/
|
|
36
|
+
eval_*.json
|
|
37
|
+
benchmark_results/
|
|
38
|
+
debug_*/
|
|
39
|
+
|
|
40
|
+
# Internal documentation (not for public repo)
|
|
41
|
+
docs/internal/
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.12
|