openadapt-ml 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. openadapt_ml-0.1.0/.env.example +132 -0
  2. openadapt_ml-0.1.0/.github/workflows/publish.yml +30 -0
  3. openadapt_ml-0.1.0/.gitignore +41 -0
  4. openadapt_ml-0.1.0/.gitmodules +3 -0
  5. openadapt_ml-0.1.0/.python-version +1 -0
  6. openadapt_ml-0.1.0/CLAUDE.md +620 -0
  7. openadapt_ml-0.1.0/LICENSE +21 -0
  8. openadapt_ml-0.1.0/PKG-INFO +818 -0
  9. openadapt_ml-0.1.0/README.md +771 -0
  10. openadapt_ml-0.1.0/configs/qwen2_5vl_synthetic.yaml +25 -0
  11. openadapt_ml-0.1.0/configs/qwen3vl_capture.yaml +31 -0
  12. openadapt_ml-0.1.0/configs/qwen3vl_capture_4bit.yaml +30 -0
  13. openadapt_ml-0.1.0/configs/qwen3vl_capture_batched.yaml +32 -0
  14. openadapt_ml-0.1.0/configs/qwen3vl_synthetic.yaml +29 -0
  15. openadapt_ml-0.1.0/configs/qwen3vl_synthetic_coord_v2.yaml +30 -0
  16. openadapt_ml-0.1.0/configs/qwen3vl_synthetic_dev.yaml +30 -0
  17. openadapt_ml-0.1.0/configs/qwen3vl_synthetic_registration_som.yaml +31 -0
  18. openadapt_ml-0.1.0/configs/qwen3vl_synthetic_som.yaml +30 -0
  19. openadapt_ml-0.1.0/docs/NEXT_STEPS_GROUNDING_ARCHITECTURE.md +336 -0
  20. openadapt_ml-0.1.0/docs/PRIVACY_IMPLEMENTATION_PLAN.md +293 -0
  21. openadapt_ml-0.1.0/docs/RECORD_IMPLEMENTATION_PLAN.md +376 -0
  22. openadapt_ml-0.1.0/docs/auto_shutoff_design.md +129 -0
  23. openadapt_ml-0.1.0/docs/azure_acr_authentication.md +159 -0
  24. openadapt_ml-0.1.0/docs/azure_waa_setup.md +308 -0
  25. openadapt_ml-0.1.0/docs/batching_and_schedulers.md +351 -0
  26. openadapt_ml-0.1.0/docs/benchmark_integration_plan.md +677 -0
  27. openadapt_ml-0.1.0/docs/benchmark_next_steps.md +105 -0
  28. openadapt_ml-0.1.0/docs/benchmark_viewer_integration.md +200 -0
  29. openadapt_ml-0.1.0/docs/benchmark_viewer_phase2.md +343 -0
  30. openadapt_ml-0.1.0/docs/cloud_gpu_training.md +124 -0
  31. openadapt_ml-0.1.0/docs/dashboard_architecture.md +77 -0
  32. openadapt_ml-0.1.0/docs/design.md +569 -0
  33. openadapt_ml-0.1.0/docs/early_termination.md +230 -0
  34. openadapt_ml-0.1.0/docs/eval_json_schema.md +361 -0
  35. openadapt_ml-0.1.0/docs/gemini_grounding.md +313 -0
  36. openadapt_ml-0.1.0/docs/gui_actor_integration.md +131 -0
  37. openadapt_ml-0.1.0/docs/images/dashboard/training_bottom.png +0 -0
  38. openadapt_ml-0.1.0/docs/images/dashboard/training_top.png +0 -0
  39. openadapt_ml-0.1.0/docs/images/dashboard/viewer_bottom.png +0 -0
  40. openadapt_ml-0.1.0/docs/images/dashboard/viewer_top.png +0 -0
  41. openadapt_ml-0.1.0/docs/images/grounding_demo.png +0 -0
  42. openadapt_ml-0.1.0/docs/images/grounding_demo_full.png +0 -0
  43. openadapt_ml-0.1.0/docs/images/training-dashboard.png +0 -0
  44. openadapt_ml-0.1.0/docs/images/viewer-comparison.png +0 -0
  45. openadapt_ml-0.1.0/docs/live_inference_design.md +340 -0
  46. openadapt_ml-0.1.0/docs/opencua_integration.md +581 -0
  47. openadapt_ml-0.1.0/docs/output_artifacts_and_media.md +123 -0
  48. openadapt_ml-0.1.0/docs/prediction_loading_architecture.md +215 -0
  49. openadapt_ml-0.1.0/docs/qwen_login_experiment.md +729 -0
  50. openadapt_ml-0.1.0/docs/roadmap.md +395 -0
  51. openadapt_ml-0.1.0/docs/set_of_marks_implementation.md +324 -0
  52. openadapt_ml-0.1.0/docs/som_implementation_verification.md +368 -0
  53. openadapt_ml-0.1.0/docs/state_and_next_steps_qwen_login.md +604 -0
  54. openadapt_ml-0.1.0/docs/stub_training_adapter.md +132 -0
  55. openadapt_ml-0.1.0/docs/synthetic_login_jitter_and_ablation.md +162 -0
  56. openadapt_ml-0.1.0/docs/training_feedback_ux.md +225 -0
  57. openadapt_ml-0.1.0/docs/unified_compute_architecture.md +225 -0
  58. openadapt_ml-0.1.0/docs/viewer_eval_integration.md +169 -0
  59. openadapt_ml-0.1.0/docs/viewer_layout_redesign.md +137 -0
  60. openadapt_ml-0.1.0/docs/vision.md +71 -0
  61. openadapt_ml-0.1.0/docs/wandb_integration.md +327 -0
  62. openadapt_ml-0.1.0/examples/README.md +64 -0
  63. openadapt_ml-0.1.0/examples/sample_data.json +97 -0
  64. openadapt_ml-0.1.0/examples/test_gemini_grounding.py +113 -0
  65. openadapt_ml-0.1.0/examples/train_from_json.py +153 -0
  66. openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/media/qwen3_2b_login_demo.gif +0 -0
  67. openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/media/qwen3_2b_login_demo_session_0001.gif +0 -0
  68. openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/plots/base_vs_ft.png +0 -0
  69. openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/plots/qwen3_2b_base_vs_ft_hardened_v2.png +0 -0
  70. openadapt_ml-0.1.0/experiments/qwen_login/2b_dev/plots/qwen_vs_apis.png +0 -0
  71. openadapt_ml-0.1.0/experiments/qwen_login/8b_hero/plots/qwen3_8b_base_vs_ft_hardened_v2.png +0 -0
  72. openadapt_ml-0.1.0/experiments/qwen_login/SOM_INVESTIGATION_REPORT.md +458 -0
  73. openadapt_ml-0.1.0/experiments/qwen_login/comprehensive_comparison.png +0 -0
  74. openadapt_ml-0.1.0/experiments/qwen_login/login_demo.gif +0 -0
  75. openadapt_ml-0.1.0/experiments/qwen_login/registration_demo.gif +0 -0
  76. openadapt_ml-0.1.0/openadapt_ml/__init__.py +0 -0
  77. openadapt_ml-0.1.0/openadapt_ml/benchmarks/__init__.py +125 -0
  78. openadapt_ml-0.1.0/openadapt_ml/benchmarks/agent.py +825 -0
  79. openadapt_ml-0.1.0/openadapt_ml/benchmarks/azure.py +761 -0
  80. openadapt_ml-0.1.0/openadapt_ml/benchmarks/base.py +366 -0
  81. openadapt_ml-0.1.0/openadapt_ml/benchmarks/cli.py +884 -0
  82. openadapt_ml-0.1.0/openadapt_ml/benchmarks/data_collection.py +432 -0
  83. openadapt_ml-0.1.0/openadapt_ml/benchmarks/runner.py +381 -0
  84. openadapt_ml-0.1.0/openadapt_ml/benchmarks/waa.py +704 -0
  85. openadapt_ml-0.1.0/openadapt_ml/cloud/__init__.py +5 -0
  86. openadapt_ml-0.1.0/openadapt_ml/cloud/azure_inference.py +441 -0
  87. openadapt_ml-0.1.0/openadapt_ml/cloud/lambda_labs.py +2445 -0
  88. openadapt_ml-0.1.0/openadapt_ml/cloud/local.py +790 -0
  89. openadapt_ml-0.1.0/openadapt_ml/config.py +56 -0
  90. openadapt_ml-0.1.0/openadapt_ml/datasets/__init__.py +0 -0
  91. openadapt_ml-0.1.0/openadapt_ml/datasets/next_action.py +507 -0
  92. openadapt_ml-0.1.0/openadapt_ml/evals/__init__.py +23 -0
  93. openadapt_ml-0.1.0/openadapt_ml/evals/grounding.py +241 -0
  94. openadapt_ml-0.1.0/openadapt_ml/evals/plot_eval_metrics.py +174 -0
  95. openadapt_ml-0.1.0/openadapt_ml/evals/trajectory_matching.py +486 -0
  96. openadapt_ml-0.1.0/openadapt_ml/grounding/__init__.py +45 -0
  97. openadapt_ml-0.1.0/openadapt_ml/grounding/base.py +236 -0
  98. openadapt_ml-0.1.0/openadapt_ml/grounding/detector.py +570 -0
  99. openadapt_ml-0.1.0/openadapt_ml/ingest/__init__.py +43 -0
  100. openadapt_ml-0.1.0/openadapt_ml/ingest/capture.py +312 -0
  101. openadapt_ml-0.1.0/openadapt_ml/ingest/loader.py +232 -0
  102. openadapt_ml-0.1.0/openadapt_ml/ingest/synthetic.py +1102 -0
  103. openadapt_ml-0.1.0/openadapt_ml/models/__init__.py +0 -0
  104. openadapt_ml-0.1.0/openadapt_ml/models/api_adapter.py +171 -0
  105. openadapt_ml-0.1.0/openadapt_ml/models/base_adapter.py +59 -0
  106. openadapt_ml-0.1.0/openadapt_ml/models/dummy_adapter.py +42 -0
  107. openadapt_ml-0.1.0/openadapt_ml/models/qwen_vl.py +426 -0
  108. openadapt_ml-0.1.0/openadapt_ml/runtime/__init__.py +0 -0
  109. openadapt_ml-0.1.0/openadapt_ml/runtime/policy.py +182 -0
  110. openadapt_ml-0.1.0/openadapt_ml/schemas/__init__.py +53 -0
  111. openadapt_ml-0.1.0/openadapt_ml/schemas/sessions.py +122 -0
  112. openadapt_ml-0.1.0/openadapt_ml/schemas/validation.py +252 -0
  113. openadapt_ml-0.1.0/openadapt_ml/scripts/__init__.py +0 -0
  114. openadapt_ml-0.1.0/openadapt_ml/scripts/compare.py +1490 -0
  115. openadapt_ml-0.1.0/openadapt_ml/scripts/demo_policy.py +62 -0
  116. openadapt_ml-0.1.0/openadapt_ml/scripts/eval_policy.py +287 -0
  117. openadapt_ml-0.1.0/openadapt_ml/scripts/make_gif.py +153 -0
  118. openadapt_ml-0.1.0/openadapt_ml/scripts/prepare_synthetic.py +43 -0
  119. openadapt_ml-0.1.0/openadapt_ml/scripts/run_qwen_login_benchmark.py +192 -0
  120. openadapt_ml-0.1.0/openadapt_ml/scripts/train.py +174 -0
  121. openadapt_ml-0.1.0/openadapt_ml/training/__init__.py +0 -0
  122. openadapt_ml-0.1.0/openadapt_ml/training/benchmark_viewer.py +1538 -0
  123. openadapt_ml-0.1.0/openadapt_ml/training/shared_ui.py +157 -0
  124. openadapt_ml-0.1.0/openadapt_ml/training/stub_provider.py +276 -0
  125. openadapt_ml-0.1.0/openadapt_ml/training/trainer.py +2446 -0
  126. openadapt_ml-0.1.0/openadapt_ml/training/viewer.py +2970 -0
  127. openadapt_ml-0.1.0/pyproject.toml +68 -0
  128. openadapt_ml-0.1.0/scripts/fix_acr_auth.py +212 -0
  129. openadapt_ml-0.1.0/scripts/setup_azure.py +1158 -0
  130. openadapt_ml-0.1.0/tests/__init__.py +0 -0
  131. openadapt_ml-0.1.0/tests/benchmarks/__init__.py +0 -0
  132. openadapt_ml-0.1.0/tests/benchmarks/test_api_agent.py +370 -0
  133. openadapt_ml-0.1.0/tests/benchmarks/test_waa.py +198 -0
  134. openadapt_ml-0.1.0/tests/test_action_parsing.py +138 -0
  135. openadapt_ml-0.1.0/tests/test_api_adapter.py +60 -0
  136. openadapt_ml-0.1.0/tests/test_local_cli.py +257 -0
  137. openadapt_ml-0.1.0/tests/test_training_dummy.py +26 -0
  138. openadapt_ml-0.1.0/uv.lock +3216 -0
@@ -0,0 +1,132 @@
1
+ # API Keys for VLM backends
2
+ ANTHROPIC_API_KEY=your-anthropic-api-key-here
3
+ OPENAI_API_KEY=your-openai-api-key-here
4
+
5
+ # =============================================================================
6
+ # Lambda Labs (Cloud GPU for fast training)
7
+ # =============================================================================
8
+ # Get your API key at: https://cloud.lambdalabs.com/api-keys
9
+ #
10
+ # Usage:
11
+ # python -m openadapt_ml.cloud.lambda_labs list # See available GPUs
12
+ # python -m openadapt_ml.cloud.lambda_labs launch # Launch A100 instance
13
+ # python -m openadapt_ml.cloud.lambda_labs terminate <id> # Stop billing!
14
+ #
15
+ LAMBDA_API_KEY=your-lambda-api-key-here
16
+
17
+ # Google Gemini API (for GeminiGrounder element detection)
18
+ # Get your key via one of these methods:
19
+ #
20
+ # Option 1: Google AI Studio (easiest)
21
+ # 1. Go to https://aistudio.google.com/apikey
22
+ # 2. Click "Create API Key"
23
+ # 3. Select or create a Google Cloud project
24
+ # 4. Copy the key
25
+ #
26
+ # Option 2: Google Cloud Console
27
+ # 1. Go to https://console.cloud.google.com/apis/credentials
28
+ # 2. Select your project
29
+ # 3. Click "Create Credentials" → "API Key"
30
+ # 4. Enable "Generative Language API" at:
31
+ # https://console.cloud.google.com/apis/library/generativelanguage.googleapis.com
32
+ #
33
+ GOOGLE_API_KEY=your-google-api-key-here
34
+
35
+ # =============================================================================
36
+ # Azure Credentials (for WAA benchmark evaluation on Azure VMs)
37
+ # =============================================================================
38
+ #
39
+ # EASY SETUP: Run the automated setup script:
40
+ # python scripts/setup_azure.py
41
+ #
42
+ # This script will:
43
+ # - Check Azure CLI installation
44
+ # - Log you in (opens browser)
45
+ # - Create service principal, resource group, and ML workspace
46
+ # - Write credentials to this file
47
+ #
48
+ # MANUAL SETUP: Follow the steps below if you prefer manual control.
49
+ #
50
+ # Azure authentication uses Service Principal credentials. To create them:
51
+ #
52
+ # Step 1: Install Azure CLI
53
+ # macOS: brew install azure-cli
54
+ # Windows: winget install Microsoft.AzureCLI
55
+ # Linux: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
56
+ #
57
+ # Step 2: Login to Azure
58
+ # az login
59
+ #
60
+ # Step 3: Get your subscription ID
61
+ # az account show --query id -o tsv
62
+ #
63
+ # Step 4: Create a Service Principal (replace <subscription-id>)
64
+ # az ad sp create-for-rbac \
65
+ # --name "openadapt-ml-waa" \
66
+ # --role "Contributor" \
67
+ # --scopes "/subscriptions/<subscription-id>" \
68
+ # --sdk-auth
69
+ #
70
+ # This outputs JSON with clientId, clientSecret, tenantId - copy these below.
71
+ #
72
+ # Step 5: Create Azure ML Workspace (if you don't have one)
73
+ # az ml workspace create \
74
+ # --name agents_ml \
75
+ # --resource-group agents \
76
+ # --location eastus
77
+ #
78
+ # Step 6: Request vCPU quota increase (if needed)
79
+ # Go to: https://portal.azure.com/#view/Microsoft_Azure_Capacity/QuotaMenuBlade
80
+ # Select your subscription → Compute → Standard Dv3 Family
81
+ # Request increase to at least 320 vCPUs (for 40 workers × 8 vCPUs each)
82
+ #
83
+ # Alternative: Use Azure CLI login (no service principal needed)
84
+ # If AZURE_CLIENT_ID is not set, DefaultAzureCredential will try:
85
+ # 1. Environment variables (service principal)
86
+ # 2. Azure CLI credentials (az login)
87
+ # 3. Managed Identity (when running on Azure)
88
+ #
89
+ # Service Principal credentials (from Step 4 output)
90
+ AZURE_CLIENT_ID=your-client-id-here
91
+ AZURE_CLIENT_SECRET=your-client-secret-here
92
+ AZURE_TENANT_ID=your-tenant-id-here
93
+
94
+ # Azure ML Workspace config
95
+ AZURE_SUBSCRIPTION_ID=your-subscription-id-here
96
+ AZURE_ML_RESOURCE_GROUP=agents
97
+ AZURE_ML_WORKSPACE_NAME=agents_ml
98
+
99
+ # Optional: Override default VM settings
100
+ # AZURE_VM_SIZE=Standard_D8_v3
101
+ # AZURE_DOCKER_IMAGE=ghcr.io/microsoft/windowsagentarena:latest
102
+
103
+ # =============================================================================
104
+ # Azure Storage for Async Inference Queue (Phase 2)
105
+ # =============================================================================
106
+ #
107
+ # These settings are auto-configured by setup_azure.py (steps 11-12).
108
+ # They enable live inference during training:
109
+ # - Training uploads checkpoints to blob storage
110
+ # - Inference worker polls queue for new checkpoints
111
+ # - Results uploaded to blob for dashboard to display
112
+ #
113
+ # Setup: Run 'python scripts/setup_azure.py' to create storage account and queue
114
+ #
115
+ # Manual setup (advanced):
116
+ # 1. Create storage account:
117
+ # az storage account create --name openadaptmlstorage \
118
+ # --resource-group agents --location eastus --sku Standard_LRS
119
+ #
120
+ # 2. Get connection string:
121
+ # az storage account show-connection-string \
122
+ # --name openadaptmlstorage -o tsv
123
+ #
124
+ # 3. Create queue and containers:
125
+ # az storage queue create --name inference-jobs --connection-string <conn-str>
126
+ # az storage container create --name checkpoints --connection-string <conn-str>
127
+ # az storage container create --name comparisons --connection-string <conn-str>
128
+ #
129
+ AZURE_STORAGE_CONNECTION_STRING=your-storage-connection-string-here
130
+ AZURE_INFERENCE_QUEUE_NAME=inference-jobs
131
+ AZURE_CHECKPOINTS_CONTAINER=checkpoints
132
+ AZURE_COMPARISONS_CONTAINER=comparisons
@@ -0,0 +1,30 @@
1
+ name: Publish to PyPI
2
+
3
+ on:
4
+ push:
5
+ tags:
6
+ - 'v*'
7
+
8
+ jobs:
9
+ publish:
10
+ runs-on: ubuntu-latest
11
+ permissions:
12
+ id-token: write # Required for trusted publishing
13
+ contents: read # Required for checkout
14
+
15
+ steps:
16
+ - uses: actions/checkout@v4
17
+
18
+ - name: Install uv
19
+ uses: astral-sh/setup-uv@v4
20
+ with:
21
+ version: "latest"
22
+
23
+ - name: Set up Python
24
+ run: uv python install 3.12
25
+
26
+ - name: Build package
27
+ run: uv build
28
+
29
+ - name: Publish to PyPI
30
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,41 @@
1
+ # Python-generated files
2
+ __pycache__/
3
+ *.py[oc]
4
+ build/
5
+ dist/
6
+ wheels/
7
+ *.egg-info
8
+
9
+ # Virtual environments
10
+ .venv
11
+ local_context_openadapt_ml_internal.md
12
+
13
+ # Environment variables
14
+ .env
15
+
16
+ # Ephemeral synthetic assets (frames, debug sessions, etc.)
17
+ synthetic/
18
+ synthetic_train/
19
+ synthetic_train_dev/
20
+
21
+ # Local OS cruft
22
+ .DS_Store
23
+
24
+ # Editor files
25
+ *.swp
26
+ *.swo
27
+ *.swn
28
+ *~
29
+ .*.sw?
30
+
31
+ # Ephemeral training/eval artifacts
32
+ logs/
33
+ checkpoints/
34
+ checkpoints_*/
35
+ /plots/
36
+ eval_*.json
37
+ benchmark_results/
38
+ debug_*/
39
+
40
+ # Internal documentation (not for public repo)
41
+ docs/internal/
@@ -0,0 +1,3 @@
1
+ [submodule "vendor/WindowsAgentArena"]
2
+ path = vendor/WindowsAgentArena
3
+ url = https://github.com/microsoft/WindowsAgentArena.git
@@ -0,0 +1 @@
1
+ 3.12