openadapt-ml 0.1.0__py3-none-any.whl → 0.2.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (112) hide show
  1. openadapt_ml/baselines/__init__.py +121 -0
  2. openadapt_ml/baselines/adapter.py +185 -0
  3. openadapt_ml/baselines/cli.py +314 -0
  4. openadapt_ml/baselines/config.py +448 -0
  5. openadapt_ml/baselines/parser.py +922 -0
  6. openadapt_ml/baselines/prompts.py +787 -0
  7. openadapt_ml/benchmarks/__init__.py +13 -107
  8. openadapt_ml/benchmarks/agent.py +297 -374
  9. openadapt_ml/benchmarks/azure.py +62 -24
  10. openadapt_ml/benchmarks/azure_ops_tracker.py +521 -0
  11. openadapt_ml/benchmarks/cli.py +1874 -751
  12. openadapt_ml/benchmarks/trace_export.py +631 -0
  13. openadapt_ml/benchmarks/viewer.py +1236 -0
  14. openadapt_ml/benchmarks/vm_monitor.py +1111 -0
  15. openadapt_ml/benchmarks/waa_deploy/Dockerfile +216 -0
  16. openadapt_ml/benchmarks/waa_deploy/__init__.py +10 -0
  17. openadapt_ml/benchmarks/waa_deploy/api_agent.py +540 -0
  18. openadapt_ml/benchmarks/waa_deploy/start_waa_server.bat +53 -0
  19. openadapt_ml/cloud/azure_inference.py +3 -5
  20. openadapt_ml/cloud/lambda_labs.py +722 -307
  21. openadapt_ml/cloud/local.py +3194 -89
  22. openadapt_ml/cloud/ssh_tunnel.py +595 -0
  23. openadapt_ml/datasets/next_action.py +125 -96
  24. openadapt_ml/evals/grounding.py +32 -9
  25. openadapt_ml/evals/plot_eval_metrics.py +15 -13
  26. openadapt_ml/evals/trajectory_matching.py +120 -57
  27. openadapt_ml/experiments/demo_prompt/__init__.py +19 -0
  28. openadapt_ml/experiments/demo_prompt/format_demo.py +236 -0
  29. openadapt_ml/experiments/demo_prompt/results/experiment_20251231_002125.json +83 -0
  30. openadapt_ml/experiments/demo_prompt/results/experiment_n30_20251231_165958.json +1100 -0
  31. openadapt_ml/experiments/demo_prompt/results/multistep_20251231_025051.json +182 -0
  32. openadapt_ml/experiments/demo_prompt/run_experiment.py +541 -0
  33. openadapt_ml/experiments/representation_shootout/__init__.py +70 -0
  34. openadapt_ml/experiments/representation_shootout/conditions.py +708 -0
  35. openadapt_ml/experiments/representation_shootout/config.py +390 -0
  36. openadapt_ml/experiments/representation_shootout/evaluator.py +659 -0
  37. openadapt_ml/experiments/representation_shootout/runner.py +687 -0
  38. openadapt_ml/experiments/waa_demo/__init__.py +10 -0
  39. openadapt_ml/experiments/waa_demo/demos.py +357 -0
  40. openadapt_ml/experiments/waa_demo/runner.py +732 -0
  41. openadapt_ml/experiments/waa_demo/tasks.py +151 -0
  42. openadapt_ml/export/__init__.py +9 -0
  43. openadapt_ml/export/__main__.py +6 -0
  44. openadapt_ml/export/cli.py +89 -0
  45. openadapt_ml/export/parquet.py +277 -0
  46. openadapt_ml/grounding/detector.py +18 -14
  47. openadapt_ml/ingest/__init__.py +11 -10
  48. openadapt_ml/ingest/capture.py +97 -86
  49. openadapt_ml/ingest/loader.py +120 -69
  50. openadapt_ml/ingest/synthetic.py +344 -193
  51. openadapt_ml/models/api_adapter.py +14 -4
  52. openadapt_ml/models/base_adapter.py +10 -2
  53. openadapt_ml/models/providers/__init__.py +288 -0
  54. openadapt_ml/models/providers/anthropic.py +266 -0
  55. openadapt_ml/models/providers/base.py +299 -0
  56. openadapt_ml/models/providers/google.py +376 -0
  57. openadapt_ml/models/providers/openai.py +342 -0
  58. openadapt_ml/models/qwen_vl.py +46 -19
  59. openadapt_ml/perception/__init__.py +35 -0
  60. openadapt_ml/perception/integration.py +399 -0
  61. openadapt_ml/retrieval/README.md +226 -0
  62. openadapt_ml/retrieval/USAGE.md +391 -0
  63. openadapt_ml/retrieval/__init__.py +91 -0
  64. openadapt_ml/retrieval/demo_retriever.py +843 -0
  65. openadapt_ml/retrieval/embeddings.py +630 -0
  66. openadapt_ml/retrieval/index.py +194 -0
  67. openadapt_ml/retrieval/retriever.py +162 -0
  68. openadapt_ml/runtime/__init__.py +50 -0
  69. openadapt_ml/runtime/policy.py +27 -14
  70. openadapt_ml/runtime/safety_gate.py +471 -0
  71. openadapt_ml/schema/__init__.py +113 -0
  72. openadapt_ml/schema/converters.py +588 -0
  73. openadapt_ml/schema/episode.py +470 -0
  74. openadapt_ml/scripts/capture_screenshots.py +530 -0
  75. openadapt_ml/scripts/compare.py +102 -61
  76. openadapt_ml/scripts/demo_policy.py +4 -1
  77. openadapt_ml/scripts/eval_policy.py +19 -14
  78. openadapt_ml/scripts/make_gif.py +1 -1
  79. openadapt_ml/scripts/prepare_synthetic.py +16 -17
  80. openadapt_ml/scripts/train.py +98 -75
  81. openadapt_ml/segmentation/README.md +920 -0
  82. openadapt_ml/segmentation/__init__.py +97 -0
  83. openadapt_ml/segmentation/adapters/__init__.py +5 -0
  84. openadapt_ml/segmentation/adapters/capture_adapter.py +420 -0
  85. openadapt_ml/segmentation/annotator.py +610 -0
  86. openadapt_ml/segmentation/cache.py +290 -0
  87. openadapt_ml/segmentation/cli.py +674 -0
  88. openadapt_ml/segmentation/deduplicator.py +656 -0
  89. openadapt_ml/segmentation/frame_describer.py +788 -0
  90. openadapt_ml/segmentation/pipeline.py +340 -0
  91. openadapt_ml/segmentation/schemas.py +622 -0
  92. openadapt_ml/segmentation/segment_extractor.py +634 -0
  93. openadapt_ml/training/azure_ops_viewer.py +1097 -0
  94. openadapt_ml/training/benchmark_viewer.py +3255 -19
  95. openadapt_ml/training/shared_ui.py +7 -7
  96. openadapt_ml/training/stub_provider.py +57 -35
  97. openadapt_ml/training/trainer.py +255 -441
  98. openadapt_ml/training/trl_trainer.py +403 -0
  99. openadapt_ml/training/viewer.py +323 -108
  100. openadapt_ml/training/viewer_components.py +180 -0
  101. {openadapt_ml-0.1.0.dist-info → openadapt_ml-0.2.1.dist-info}/METADATA +312 -69
  102. openadapt_ml-0.2.1.dist-info/RECORD +116 -0
  103. openadapt_ml/benchmarks/base.py +0 -366
  104. openadapt_ml/benchmarks/data_collection.py +0 -432
  105. openadapt_ml/benchmarks/runner.py +0 -381
  106. openadapt_ml/benchmarks/waa.py +0 -704
  107. openadapt_ml/schemas/__init__.py +0 -53
  108. openadapt_ml/schemas/sessions.py +0 -122
  109. openadapt_ml/schemas/validation.py +0 -252
  110. openadapt_ml-0.1.0.dist-info/RECORD +0 -55
  111. {openadapt_ml-0.1.0.dist-info → openadapt_ml-0.2.1.dist-info}/WHEEL +0 -0
  112. {openadapt_ml-0.1.0.dist-info → openadapt_ml-0.2.1.dist-info}/licenses/LICENSE +0 -0
@@ -13,7 +13,7 @@ def get_shared_header_css() -> str:
13
13
  This CSS is used by both the Training Dashboard and the Viewer.
14
14
  Any changes here will affect all dashboards consistently.
15
15
  """
16
- return '''
16
+ return """
17
17
  .unified-header {
18
18
  display: flex;
19
19
  align-items: center;
@@ -101,7 +101,7 @@ def get_shared_header_css() -> str:
101
101
  color: var(--text-muted);
102
102
  font-family: "SF Mono", Monaco, monospace;
103
103
  }
104
- '''
104
+ """
105
105
 
106
106
 
107
107
  def generate_shared_header_html(
@@ -125,14 +125,14 @@ def generate_shared_header_html(
125
125
 
126
126
  controls_section = ""
127
127
  if controls_html or meta_html:
128
- controls_section = f'''
128
+ controls_section = f"""
129
129
  <div class="controls-section">
130
130
  {controls_html}
131
- {f'<span class="header-meta">{meta_html}</span>' if meta_html else ''}
131
+ {f'<span class="header-meta">{meta_html}</span>' if meta_html else ""}
132
132
  </div>
133
- '''
133
+ """
134
134
 
135
- return f'''
135
+ return f"""
136
136
  <div class="unified-header">
137
137
  <div class="nav-tabs">
138
138
  <a href="dashboard.html" class="nav-tab {training_active}">Training</a>
@@ -141,7 +141,7 @@ def generate_shared_header_html(
141
141
  </div>
142
142
  {controls_section}
143
143
  </div>
144
- '''
144
+ """
145
145
 
146
146
 
147
147
  def build_nav_links() -> list[tuple[str, str]]:
@@ -2,7 +2,6 @@
2
2
 
3
3
  import json
4
4
  import random
5
- import sys
6
5
  import time
7
6
  from datetime import datetime
8
7
  from pathlib import Path
@@ -92,13 +91,15 @@ class StubTrainingProvider:
92
91
 
93
92
  elapsed = time.time() - self.start_time
94
93
 
95
- self.losses.append({
96
- "epoch": self.current_epoch,
97
- "step": self.current_step + 1,
98
- "loss": loss,
99
- "lr": 5e-5,
100
- "time": elapsed,
101
- })
94
+ self.losses.append(
95
+ {
96
+ "epoch": self.current_epoch,
97
+ "step": self.current_step + 1,
98
+ "loss": loss,
99
+ "lr": 5e-5,
100
+ "time": elapsed,
101
+ }
102
+ )
102
103
 
103
104
  self.current_step += 1
104
105
 
@@ -123,32 +124,37 @@ class StubTrainingProvider:
123
124
  if not sample_path.exists():
124
125
  # Try to copy from common capture location
125
126
  import shutil
126
- capture_screenshots = Path.home() / "oa/src/openadapt-capture/turn-off-nightshift/screenshots"
127
+
128
+ capture_screenshots = (
129
+ Path.home() / "oa/src/openadapt-capture/turn-off-nightshift/screenshots"
130
+ )
127
131
  if capture_screenshots.exists():
128
132
  sample_path.parent.mkdir(parents=True, exist_ok=True)
129
133
  for img in capture_screenshots.glob("*.png"):
130
134
  shutil.copy(img, sample_path)
131
135
  break # Just copy the first one
132
136
 
133
- self.evaluations.append({
134
- "epoch": self.current_epoch,
135
- "sample_idx": 7, # Match the real training sample
136
- "image_path": "screenshots/sample.png",
137
- "human_action": {
138
- "type": "click",
139
- "x": 0.65,
140
- "y": 0.65,
141
- "text": None,
142
- },
143
- "predicted_action": {
144
- "type": "click",
145
- "x": 0.65 + random.uniform(-0.15, 0.15) * (1 - accuracy_boost),
146
- "y": 0.65 + random.uniform(-0.15, 0.15) * (1 - accuracy_boost),
147
- "raw_output": f"Thought: [Stub] Epoch {self.current_epoch} - analyzing screenshot to find target element. The model is learning to identify UI components.\nAction: CLICK(x=0.65, y=0.65)",
148
- },
149
- "distance": random.uniform(0.05, 0.2) * (1 - accuracy_boost),
150
- "correct": random.random() > (0.5 - accuracy_boost),
151
- })
137
+ self.evaluations.append(
138
+ {
139
+ "epoch": self.current_epoch,
140
+ "sample_idx": 7, # Match the real training sample
141
+ "image_path": "screenshots/sample.png",
142
+ "human_action": {
143
+ "type": "click",
144
+ "x": 0.65,
145
+ "y": 0.65,
146
+ "text": None,
147
+ },
148
+ "predicted_action": {
149
+ "type": "click",
150
+ "x": 0.65 + random.uniform(-0.15, 0.15) * (1 - accuracy_boost),
151
+ "y": 0.65 + random.uniform(-0.15, 0.15) * (1 - accuracy_boost),
152
+ "raw_output": f"Thought: [Stub] Epoch {self.current_epoch} - analyzing screenshot to find target element. The model is learning to identify UI components.\nAction: CLICK(x=0.65, y=0.65)",
153
+ },
154
+ "distance": random.uniform(0.05, 0.2) * (1 - accuracy_boost),
155
+ "correct": random.random() > (0.5 - accuracy_boost),
156
+ }
157
+ )
152
158
 
153
159
  def get_status(self) -> dict:
154
160
  """Return current training status.
@@ -161,7 +167,11 @@ class StubTrainingProvider:
161
167
 
162
168
  # Determine status
163
169
  if self.termination_status:
164
- status = "completed" if self.termination_status == "auto_complete" else self.termination_status
170
+ status = (
171
+ "completed"
172
+ if self.termination_status == "auto_complete"
173
+ else self.termination_status
174
+ )
165
175
  elif self.is_complete():
166
176
  status = "completed"
167
177
  else:
@@ -215,11 +225,17 @@ class StubTrainingProvider:
215
225
  Args:
216
226
  callback: Optional function called after each step with status dict
217
227
  """
218
- self._log(f"[Stub] Starting simulated training: {self.epochs} epochs, {self.steps_per_epoch} steps/epoch")
228
+ self._log(
229
+ f"[Stub] Starting simulated training: {self.epochs} epochs, {self.steps_per_epoch} steps/epoch"
230
+ )
219
231
  self._log(f"[Stub] Output: {self.output_dir}")
220
- self._log(f"[Stub] Step delay: {self.step_delay}s (total ~{self.epochs * self.steps_per_epoch * self.step_delay:.0f}s)")
232
+ self._log(
233
+ f"[Stub] Step delay: {self.step_delay}s (total ~{self.epochs * self.steps_per_epoch * self.step_delay:.0f}s)"
234
+ )
221
235
  if self.early_stop_loss > 0:
222
- self._log(f"[Stub] Early stop: loss < {self.early_stop_loss} for {self.early_stop_patience} steps")
236
+ self._log(
237
+ f"[Stub] Early stop: loss < {self.early_stop_loss} for {self.early_stop_patience} steps"
238
+ )
223
239
  self._log("")
224
240
 
225
241
  while not self.is_complete():
@@ -239,9 +255,13 @@ class StubTrainingProvider:
239
255
  if self.early_stop_loss > 0 and loss < self.early_stop_loss:
240
256
  self.consecutive_low_loss += 1
241
257
  if self.consecutive_low_loss >= self.early_stop_patience:
242
- self._log(f"\n[Stub] Auto-stopped: loss ({loss:.4f}) < {self.early_stop_loss} for {self.early_stop_patience} steps")
258
+ self._log(
259
+ f"\n[Stub] Auto-stopped: loss ({loss:.4f}) < {self.early_stop_loss} for {self.early_stop_patience} steps"
260
+ )
243
261
  self.termination_status = "auto_low_loss"
244
- self.termination_message = f"Loss reached {loss:.4f} (< {self.early_stop_loss})"
262
+ self.termination_message = (
263
+ f"Loss reached {loss:.4f} (< {self.early_stop_loss})"
264
+ )
245
265
  self.write_status()
246
266
  break
247
267
  else:
@@ -253,7 +273,9 @@ class StubTrainingProvider:
253
273
  epoch = status["epoch"]
254
274
  step = status["step"]
255
275
  display_epoch = min(epoch + 1, self.epochs) # Cap at max for display
256
- self._log(f" Epoch {display_epoch}/{self.epochs} | Step {step} | Loss: {loss:.4f}")
276
+ self._log(
277
+ f" Epoch {display_epoch}/{self.epochs} | Step {step} | Loss: {loss:.4f}"
278
+ )
257
279
 
258
280
  if callback:
259
281
  callback(status)