screen-controller-ai 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- screen_controller_ai-0.1.2/PKG-INFO +59 -0
- screen_controller_ai-0.1.2/README.md +47 -0
- screen_controller_ai-0.1.2/controller.py +168 -0
- screen_controller_ai-0.1.2/pyproject.toml +17 -0
- screen_controller_ai-0.1.2/screen_controller_ai.egg-info/PKG-INFO +59 -0
- {screen_controller_ai-0.1.1 → screen_controller_ai-0.1.2}/screen_controller_ai.egg-info/SOURCES.txt +1 -0
- {screen_controller_ai-0.1.1 → screen_controller_ai-0.1.2}/screen_controller_ai.egg-info/requires.txt +3 -2
- screen_controller_ai-0.1.1/PKG-INFO +0 -15
- screen_controller_ai-0.1.1/controller.py +0 -117
- screen_controller_ai-0.1.1/pyproject.toml +0 -27
- screen_controller_ai-0.1.1/screen_controller_ai.egg-info/PKG-INFO +0 -15
- {screen_controller_ai-0.1.1 → screen_controller_ai-0.1.2}/screen_controller_ai.egg-info/dependency_links.txt +0 -0
- {screen_controller_ai-0.1.1 → screen_controller_ai-0.1.2}/screen_controller_ai.egg-info/top_level.txt +0 -0
- {screen_controller_ai-0.1.1 → screen_controller_ai-0.1.2}/setup.cfg +0 -0
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: screen-controller-ai
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.8
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: google-genai
|
|
9
|
+
Requires-Dist: pillow
|
|
10
|
+
Requires-Dist: pynput
|
|
11
|
+
Requires-Dist: pyautogui
|
|
12
|
+
|
|
13
|
+
# Screen Controller AI
|
|
14
|
+
|
|
15
|
+
A safe, dual-model agentic desktop controller with human-in-the-loop safety features built using Python and the Google GenAI SDK.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
Install the package via pip:
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
pip install screen-controller-ai
|
|
23
|
+
|
|
24
|
+
Quick Start Example
|
|
25
|
+
|
|
26
|
+
Here is how you use the custom interface, with every single line explained step-by-step:
|
|
27
|
+
|
|
28
|
+
Python
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
import screen_controller_ai as x # (imports the screen controller package and renames it to x for easy shorthand use)
|
|
33
|
+
|
|
34
|
+
x.allow_controller() # (enables safety monitoring so any mouse movement or keyboard click by the user instantly halts the controller and returns full control)
|
|
35
|
+
|
|
36
|
+
x.open("[https://www.duckdns.org](https://www.duckdns.org)") # (opens the target website or application URL where the task needs to be performed (you may ofcourse choose other websites))
|
|
37
|
+
|
|
38
|
+
x.key = "YOUR_GEMINI_API_KEY" # (sets the API key so the AI models can connect and run in the background)
|
|
39
|
+
|
|
40
|
+
x.target = "get a free sub domain from duck dns" # (defines the specific goal for the AI to complete while a secondary background model verifies its progress)
|
|
41
|
+
|
|
42
|
+
x.allow_screen_viewer() # (gives the AI permission to view and capture the computer screen live)
|
|
43
|
+
|
|
44
|
+
x.start() # (launches the automated process so you can watch live as the AI works toward your target)
|
|
45
|
+
Features
|
|
46
|
+
Instant Safety Interruption: Automatically stops execution the moment any human mouse movement or keyboard click is detected.
|
|
47
|
+
|
|
48
|
+
Dual-Model Verification: Uses Gemini Flash for task execution and Gemini Flash Lite in the background to verify progress.
|
|
49
|
+
|
|
50
|
+
Simple Python Interface: Clean shorthand syntax designed for quick automation tasks.
|
|
51
|
+
|
|
52
|
+
Requirements
|
|
53
|
+
Python >= 3.8
|
|
54
|
+
|
|
55
|
+
pynput
|
|
56
|
+
|
|
57
|
+
pillow
|
|
58
|
+
|
|
59
|
+
google-genai
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# Screen Controller AI
|
|
2
|
+
|
|
3
|
+
A safe, dual-model agentic desktop controller with human-in-the-loop safety features built using Python and the Google GenAI SDK.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
Install the package via pip:
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
pip install screen-controller-ai
|
|
11
|
+
|
|
12
|
+
Quick Start Example
|
|
13
|
+
|
|
14
|
+
Here is how you use the custom interface, with every single line explained step-by-step:
|
|
15
|
+
|
|
16
|
+
Python
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
import screen_controller_ai as x # (imports the screen controller package and renames it to x for easy shorthand use)
|
|
21
|
+
|
|
22
|
+
x.allow_controller() # (enables safety monitoring so any mouse movement or keyboard click by the user instantly halts the controller and returns full control)
|
|
23
|
+
|
|
24
|
+
x.open("[https://www.duckdns.org](https://www.duckdns.org)") # (opens the target website or application URL where the task needs to be performed (you may ofcourse choose other websites))
|
|
25
|
+
|
|
26
|
+
x.key = "YOUR_GEMINI_API_KEY" # (sets the API key so the AI models can connect and run in the background)
|
|
27
|
+
|
|
28
|
+
x.target = "get a free sub domain from duck dns" # (defines the specific goal for the AI to complete while a secondary background model verifies its progress)
|
|
29
|
+
|
|
30
|
+
x.allow_screen_viewer() # (gives the AI permission to view and capture the computer screen live)
|
|
31
|
+
|
|
32
|
+
x.start() # (launches the automated process so you can watch live as the AI works toward your target)
|
|
33
|
+
Features
|
|
34
|
+
Instant Safety Interruption: Automatically stops execution the moment any human mouse movement or keyboard click is detected.
|
|
35
|
+
|
|
36
|
+
Dual-Model Verification: Uses Gemini Flash for task execution and Gemini Flash Lite in the background to verify progress.
|
|
37
|
+
|
|
38
|
+
Simple Python Interface: Clean shorthand syntax designed for quick automation tasks.
|
|
39
|
+
|
|
40
|
+
Requirements
|
|
41
|
+
Python >= 3.8
|
|
42
|
+
|
|
43
|
+
pynput
|
|
44
|
+
|
|
45
|
+
pillow
|
|
46
|
+
|
|
47
|
+
google-genai
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import time
|
|
2
|
+
import threading
|
|
3
|
+
import webbrowser
|
|
4
|
+
from PIL import ImageGrab
|
|
5
|
+
import pyautogui
|
|
6
|
+
from pynput import mouse, keyboard
|
|
7
|
+
from google import genai
|
|
8
|
+
|
|
9
|
+
# Module-level state variables
|
|
10
|
+
key = None
|
|
11
|
+
target = ""
|
|
12
|
+
_controller_allowed = False
|
|
13
|
+
_screen_viewer_allowed = False
|
|
14
|
+
_is_running = False
|
|
15
|
+
_listener_active = False
|
|
16
|
+
|
|
17
|
+
# Fail-safe for Pyautogui: moving mouse to any corner instantly aborts as extra safety
|
|
18
|
+
pyautogui.FAILSAFE = True
|
|
19
|
+
|
|
20
|
+
def allow_controller():
|
|
21
|
+
"""Enables safety monitoring so any human mouse movement or keyboard click instantly halts execution."""
|
|
22
|
+
global _controller_allowed, _listener_active
|
|
23
|
+
_controller_allowed = True
|
|
24
|
+
print("[Safety] Controller safety monitor armed. Any mouse/keyboard action will halt execution.")
|
|
25
|
+
if not _listener_active:
|
|
26
|
+
_start_safety_listener()
|
|
27
|
+
|
|
28
|
+
def open(url):
|
|
29
|
+
"""Opens the specified URL in the default web browser."""
|
|
30
|
+
print(f"[Action] Opening URL: {url}")
|
|
31
|
+
webbrowser.open(url)
|
|
32
|
+
time.sleep(3) # Give browser time to load
|
|
33
|
+
|
|
34
|
+
def allow_screen_viewer():
|
|
35
|
+
"""Grants permission for the AI to capture and analyze the screen."""
|
|
36
|
+
global _screen_viewer_allowed
|
|
37
|
+
_screen_viewer_allowed = True
|
|
38
|
+
print("[Vision] Screen viewer permission granted.")
|
|
39
|
+
|
|
40
|
+
def _start_safety_listener():
|
|
41
|
+
"""Fires background listeners to detect human input and abort instantly."""
|
|
42
|
+
global _listener_active, _is_running
|
|
43
|
+
_listener_active = True
|
|
44
|
+
|
|
45
|
+
def on_move(x, y):
|
|
46
|
+
if _is_running:
|
|
47
|
+
print("\n[ALERT] Human mouse movement detected! Halting controller instantly.")
|
|
48
|
+
_stop_execution()
|
|
49
|
+
|
|
50
|
+
def on_click(x, y, button, pressed):
|
|
51
|
+
if _is_running and pressed:
|
|
52
|
+
print("\n[ALERT] Human mouse click detected! Halting controller instantly.")
|
|
53
|
+
_stop_execution()
|
|
54
|
+
|
|
55
|
+
def on_press(key_pressed):
|
|
56
|
+
if _is_running:
|
|
57
|
+
print(f"\n[ALERT] Human keyboard input detected! Halting controller instantly.")
|
|
58
|
+
_stop_execution()
|
|
59
|
+
|
|
60
|
+
mouse_listener = mouse.Listener(on_move=on_move, on_click=on_click)
|
|
61
|
+
keyboard_listener = keyboard.Listener(on_press=on_press)
|
|
62
|
+
|
|
63
|
+
mouse_listener.daemon = True
|
|
64
|
+
keyboard_listener.daemon = True
|
|
65
|
+
|
|
66
|
+
mouse_listener.start()
|
|
67
|
+
keyboard_listener.start()
|
|
68
|
+
|
|
69
|
+
def _stop_execution():
|
|
70
|
+
global _is_running
|
|
71
|
+
_is_running = False
|
|
72
|
+
|
|
73
|
+
def start():
|
|
74
|
+
"""Starts the dual-model AI loop to see the screen and control the desktop toward the target."""
|
|
75
|
+
global _is_running, key, target, _controller_allowed, _screen_viewer_allowed
|
|
76
|
+
|
|
77
|
+
if not _controller_allowed:
|
|
78
|
+
raise RuntimeError("Controller not allowed! Call x.allow_controller() first.")
|
|
79
|
+
if not _screen_viewer_allowed:
|
|
80
|
+
raise RuntimeError("Screen viewer not allowed! Call x.allow_screen_viewer() first.")
|
|
81
|
+
if not key:
|
|
82
|
+
raise ValueError("API key not set! Assign x.key.")
|
|
83
|
+
if not target:
|
|
84
|
+
raise ValueError("Target not set! Assign x.target.")
|
|
85
|
+
|
|
86
|
+
_is_running = True
|
|
87
|
+
print(f"\n[Started] Running task target: '{target}'")
|
|
88
|
+
|
|
89
|
+
client = genai.Client(api_key=key if key.startswith("AIza") else None)
|
|
90
|
+
|
|
91
|
+
step_count = 0
|
|
92
|
+
while _is_running and step_count < 10:
|
|
93
|
+
step_count += 1
|
|
94
|
+
print(f"\n--- Execution Step {step_count} ---")
|
|
95
|
+
|
|
96
|
+
# 1. Capture current desktop screen so the AI can SEE it
|
|
97
|
+
screenshot = ImageGrab.grab()
|
|
98
|
+
screenshot_path = f"screenshot_step_{step_count}.png"
|
|
99
|
+
screenshot.save(screenshot_path)
|
|
100
|
+
print(f"[Vision] Screenshot captured.")
|
|
101
|
+
|
|
102
|
+
# 2. Ask Gemini to analyze screen and return action coordinates or text to type
|
|
103
|
+
prompt = f"""
|
|
104
|
+
Goal: {target}
|
|
105
|
+
Look at the attached screen capture. Decide the next action to control the computer.
|
|
106
|
+
Format your response strictly as one of these commands:
|
|
107
|
+
- CLICK X, Y (to click specific pixel coordinates on screen)
|
|
108
|
+
- TYPE "text" (to type text using keyboard)
|
|
109
|
+
- PRESS key (to press a specific key like enter, tab)
|
|
110
|
+
- DONE (if the goal is fully achieved)
|
|
111
|
+
"""
|
|
112
|
+
|
|
113
|
+
try:
|
|
114
|
+
# Main model executing the action
|
|
115
|
+
response = client.models.generate_content(
|
|
116
|
+
model='gemini-2.5-flash',
|
|
117
|
+
contents=[prompt, screenshot]
|
|
118
|
+
)
|
|
119
|
+
action_text = response.text.strip()
|
|
120
|
+
print(f"[AI Action]: {action_text}")
|
|
121
|
+
|
|
122
|
+
if "DONE" in action_text.upper():
|
|
123
|
+
print("[Success] AI reports the task is complete!")
|
|
124
|
+
break
|
|
125
|
+
|
|
126
|
+
# Parse and physically EXECUTE the action on the user's screen
|
|
127
|
+
if "CLICK" in action_text.upper():
|
|
128
|
+
# Extract coordinates e.g., CLICK 500, 300
|
|
129
|
+
try:
|
|
130
|
+
coords = action_text.split("CLICK")[1].strip().replace("(", "").replace(")", "").split(",")
|
|
131
|
+
cx, cy = int(coords[0].strip()), int(coords[1].strip())
|
|
132
|
+
print(f"[Control] Clicking at X={cx}, Y={cy}")
|
|
133
|
+
pyautogui.click(cx, cy)
|
|
134
|
+
except Exception as ex:
|
|
135
|
+
print(f"[Control Error] Could not parse click coordinates: {ex}")
|
|
136
|
+
|
|
137
|
+
elif "TYPE" in action_text.upper():
|
|
138
|
+
# Extract text inside quotes
|
|
139
|
+
try:
|
|
140
|
+
text_to_type = action_text.split('"')[1]
|
|
141
|
+
print(f"[Control] Typing: '{text_to_type}'")
|
|
142
|
+
pyautogui.write(text_to_type, interval=0.05)
|
|
143
|
+
except Exception as ex:
|
|
144
|
+
print(f"[Control Error] Could not parse text to type: {ex}")
|
|
145
|
+
|
|
146
|
+
elif "PRESS" in action_text.upper():
|
|
147
|
+
try:
|
|
148
|
+
key_to_press = action_text.split("PRESS")[1].strip().lower()
|
|
149
|
+
print(f"[Control] Pressing key: {key_to_press}")
|
|
150
|
+
pyautogui.press(key_to_press)
|
|
151
|
+
except Exception as ex:
|
|
152
|
+
print(f"[Control Error] Could not parse key: {ex}")
|
|
153
|
+
|
|
154
|
+
# Background verifier model checking progress
|
|
155
|
+
verifier_prompt = f"Verify if the goal '{target}' is successfully progressing based on current desktop action."
|
|
156
|
+
verifier_response = client.models.generate_content(
|
|
157
|
+
model='gemini-2.5-flash-lite',
|
|
158
|
+
contents=[verifier_prompt]
|
|
159
|
+
)
|
|
160
|
+
print(f"[Background Verifier]: {verifier_response.text.strip()}")
|
|
161
|
+
|
|
162
|
+
except Exception as e:
|
|
163
|
+
print(f"[Error]: {e}")
|
|
164
|
+
break
|
|
165
|
+
|
|
166
|
+
time.sleep(3)
|
|
167
|
+
|
|
168
|
+
print("\n[Finished] Screen Controller session completed or halted.")
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "screen-controller-ai"
|
|
7
|
+
version = "0.1.2"
|
|
8
|
+
description = "A safe, dual-model agentic desktop controller with human-in-the-loop safety features."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
dependencies = [
|
|
13
|
+
"google-genai",
|
|
14
|
+
"pillow",
|
|
15
|
+
"pynput",
|
|
16
|
+
"pyautogui"
|
|
17
|
+
]
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: screen-controller-ai
|
|
3
|
+
Version: 0.1.2
|
|
4
|
+
Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features.
|
|
5
|
+
License: MIT
|
|
6
|
+
Requires-Python: >=3.8
|
|
7
|
+
Description-Content-Type: text/markdown
|
|
8
|
+
Requires-Dist: google-genai
|
|
9
|
+
Requires-Dist: pillow
|
|
10
|
+
Requires-Dist: pynput
|
|
11
|
+
Requires-Dist: pyautogui
|
|
12
|
+
|
|
13
|
+
# Screen Controller AI
|
|
14
|
+
|
|
15
|
+
A safe, dual-model agentic desktop controller with human-in-the-loop safety features built using Python and the Google GenAI SDK.
|
|
16
|
+
|
|
17
|
+
## Installation
|
|
18
|
+
|
|
19
|
+
Install the package via pip:
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
pip install screen-controller-ai
|
|
23
|
+
|
|
24
|
+
Quick Start Example
|
|
25
|
+
|
|
26
|
+
Here is how you use the custom interface, with every single line explained step-by-step:
|
|
27
|
+
|
|
28
|
+
Python
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
import screen_controller_ai as x # (imports the screen controller package and renames it to x for easy shorthand use)
|
|
33
|
+
|
|
34
|
+
x.allow_controller() # (enables safety monitoring so any mouse movement or keyboard click by the user instantly halts the controller and returns full control)
|
|
35
|
+
|
|
36
|
+
x.open("[https://www.duckdns.org](https://www.duckdns.org)") # (opens the target website or application URL where the task needs to be performed (you may ofcourse choose other websites))
|
|
37
|
+
|
|
38
|
+
x.key = "YOUR_GEMINI_API_KEY" # (sets the API key so the AI models can connect and run in the background)
|
|
39
|
+
|
|
40
|
+
x.target = "get a free sub domain from duck dns" # (defines the specific goal for the AI to complete while a secondary background model verifies its progress)
|
|
41
|
+
|
|
42
|
+
x.allow_screen_viewer() # (gives the AI permission to view and capture the computer screen live)
|
|
43
|
+
|
|
44
|
+
x.start() # (launches the automated process so you can watch live as the AI works toward your target)
|
|
45
|
+
Features
|
|
46
|
+
Instant Safety Interruption: Automatically stops execution the moment any human mouse movement or keyboard click is detected.
|
|
47
|
+
|
|
48
|
+
Dual-Model Verification: Uses Gemini Flash for task execution and Gemini Flash Lite in the background to verify progress.
|
|
49
|
+
|
|
50
|
+
Simple Python Interface: Clean shorthand syntax designed for quick automation tasks.
|
|
51
|
+
|
|
52
|
+
Requirements
|
|
53
|
+
Python >= 3.8
|
|
54
|
+
|
|
55
|
+
pynput
|
|
56
|
+
|
|
57
|
+
pillow
|
|
58
|
+
|
|
59
|
+
google-genai
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: screen-controller-ai
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features
|
|
5
|
-
Author: Atharva
|
|
6
|
-
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/atharva/screen-controller
|
|
8
|
-
Classifier: Programming Language :: Python :: 3
|
|
9
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
-
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.8
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
Requires-Dist: pynput
|
|
14
|
-
Requires-Dist: pillow
|
|
15
|
-
Requires-Dist: google-genai
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
import time
|
|
2
|
-
import threading
|
|
3
|
-
import sys
|
|
4
|
-
from pynput import mouse, keyboard
|
|
5
|
-
from PIL import ImageGrab
|
|
6
|
-
|
|
7
|
-
class ScreenController:
|
|
8
|
-
def __init__(self):
|
|
9
|
-
self.url = None
|
|
10
|
-
self.api_key = None
|
|
11
|
-
self.task_target = None
|
|
12
|
-
self.screen_viewer_allowed = False
|
|
13
|
-
self.controller_allowed = False
|
|
14
|
-
self._stop_event = threading.Event()
|
|
15
|
-
|
|
16
|
-
self._mouse_listener = None
|
|
17
|
-
self._keyboard_listener = None
|
|
18
|
-
|
|
19
|
-
def allow_controller(self):
|
|
20
|
-
"""Enables the safety interrupt listener (mouse/keyboard tripwire)."""
|
|
21
|
-
self.controller_allowed = True
|
|
22
|
-
print("[Safety] Controller armed. Move your mouse or press any key to instantly abort.")
|
|
23
|
-
self._start_safety_listeners()
|
|
24
|
-
return self
|
|
25
|
-
|
|
26
|
-
def open(self, url):
|
|
27
|
-
"""Sets the target URL to open."""
|
|
28
|
-
self.url = url
|
|
29
|
-
print(f"[Action] Target URL configured: {self.url}")
|
|
30
|
-
return self
|
|
31
|
-
|
|
32
|
-
def key(self, api_key):
|
|
33
|
-
"""Sets the API key for the AI models."""
|
|
34
|
-
self.api_key = api_key
|
|
35
|
-
print("[Config] API Key loaded successfully.")
|
|
36
|
-
return self
|
|
37
|
-
|
|
38
|
-
def target(self, goal):
|
|
39
|
-
"""Defines what the AI agent must accomplish."""
|
|
40
|
-
self.task_target = goal
|
|
41
|
-
print(f"[Target] Goal defined: '{self.task_target}'")
|
|
42
|
-
return self
|
|
43
|
-
|
|
44
|
-
def allow_screen_viewer(self):
|
|
45
|
-
"""Grants permission for the model to capture screenshots."""
|
|
46
|
-
self.screen_viewer_allowed = True
|
|
47
|
-
print("[Permission] Screen viewer enabled.")
|
|
48
|
-
return self
|
|
49
|
-
|
|
50
|
-
def _trigger_abort(self, reason="User input detected"):
|
|
51
|
-
"""Instantly stops the execution loop."""
|
|
52
|
-
if not self._stop_event.is_set():
|
|
53
|
-
print(f"\n[ABORTED] Safety tripwire triggered ({reason}). Stopping controller immediately.")
|
|
54
|
-
self._stop_event.set()
|
|
55
|
-
self._stop_safety_listeners()
|
|
56
|
-
|
|
57
|
-
def _start_safety_listeners(self):
|
|
58
|
-
"""Starts background listeners to detect user mouse movement or key presses."""
|
|
59
|
-
def on_move(x, y):
|
|
60
|
-
self._trigger_abort("Mouse moved")
|
|
61
|
-
|
|
62
|
-
def on_click(x, y, button, pressed):
|
|
63
|
-
self._trigger_abort("Mouse clicked")
|
|
64
|
-
|
|
65
|
-
def on_press(key):
|
|
66
|
-
self._trigger_abort(f"Key pressed: {key}")
|
|
67
|
-
|
|
68
|
-
# Initialize non-blocking listeners
|
|
69
|
-
self._mouse_listener = mouse.Listener(on_move=on_move, on_click=on_click)
|
|
70
|
-
self._keyboard_listener = keyboard.Listener(on_press=on_press)
|
|
71
|
-
|
|
72
|
-
self._mouse_listener.start()
|
|
73
|
-
self._keyboard_listener.start()
|
|
74
|
-
|
|
75
|
-
def _stop_safety_listeners(self):
|
|
76
|
-
"""Cleanly stops the safety listeners."""
|
|
77
|
-
if self._mouse_listener:
|
|
78
|
-
self._mouse_listener.stop()
|
|
79
|
-
if self._keyboard_listener:
|
|
80
|
-
self._keyboard_listener.stop()
|
|
81
|
-
|
|
82
|
-
def start(self):
|
|
83
|
-
"""Executes the dual-model processing and action loop."""
|
|
84
|
-
if not self.controller_allowed or not self.screen_viewer_allowed:
|
|
85
|
-
raise PermissionError("You must allow controller and screen viewer before starting!")
|
|
86
|
-
|
|
87
|
-
if not self.api_key:
|
|
88
|
-
raise ValueError("API key is missing! Set it using x.key().")
|
|
89
|
-
|
|
90
|
-
print(f"\n[Running] Starting task: '{self.task_target}'")
|
|
91
|
-
print("[Model] Main execution model active.")
|
|
92
|
-
print("[Model] Background verification model active.")
|
|
93
|
-
|
|
94
|
-
try:
|
|
95
|
-
while not self._stop_event.is_set():
|
|
96
|
-
# 1. Capture screen snapshot safely
|
|
97
|
-
if self.screen_viewer_allowed:
|
|
98
|
-
screenshot = ImageGrab.grab()
|
|
99
|
-
# (Here is where you would send the screenshot to Gemini Flash Lite)
|
|
100
|
-
|
|
101
|
-
# 2. Main Model execution step & Background Verifier check
|
|
102
|
-
print("[Looping] Analyzing screen state and verifying task progress...")
|
|
103
|
-
|
|
104
|
-
# Simulate a working interval (check every 2 seconds)
|
|
105
|
-
for _ in range(20):
|
|
106
|
-
if self._stop_event.is_set():
|
|
107
|
-
break
|
|
108
|
-
time.sleep(0.1)
|
|
109
|
-
|
|
110
|
-
# Break condition placeholder for testing
|
|
111
|
-
break
|
|
112
|
-
|
|
113
|
-
except Exception as e:
|
|
114
|
-
print(f"[Error] {e}")
|
|
115
|
-
finally:
|
|
116
|
-
self._stop_safety_listeners()
|
|
117
|
-
print("[Stopped] Screen controller session closed safely.")
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
[build-system]
|
|
2
|
-
requires = ["setuptools>=61.0.0"]
|
|
3
|
-
build-backend = "setuptools.build_meta"
|
|
4
|
-
|
|
5
|
-
[project]
|
|
6
|
-
name = "screen-controller-ai"
|
|
7
|
-
version = "0.1.1"
|
|
8
|
-
description = "A safe, dual-model agentic desktop controller with human-in-the-loop safety features"
|
|
9
|
-
readme = "README.md"
|
|
10
|
-
authors = [
|
|
11
|
-
{ name = "Atharva" }
|
|
12
|
-
]
|
|
13
|
-
license = { text = "MIT" }
|
|
14
|
-
classifiers = [
|
|
15
|
-
"Programming Language :: Python :: 3",
|
|
16
|
-
"License :: OSI Approved :: MIT License",
|
|
17
|
-
"Operating System :: OS Independent",
|
|
18
|
-
]
|
|
19
|
-
requires-python = ">=3.8"
|
|
20
|
-
dependencies = [
|
|
21
|
-
"pynput",
|
|
22
|
-
"pillow",
|
|
23
|
-
"google-genai"
|
|
24
|
-
]
|
|
25
|
-
|
|
26
|
-
[project.urls]
|
|
27
|
-
Homepage = "https://github.com/atharva/screen-controller"
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: screen-controller-ai
|
|
3
|
-
Version: 0.1.1
|
|
4
|
-
Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features
|
|
5
|
-
Author: Atharva
|
|
6
|
-
License: MIT
|
|
7
|
-
Project-URL: Homepage, https://github.com/atharva/screen-controller
|
|
8
|
-
Classifier: Programming Language :: Python :: 3
|
|
9
|
-
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
-
Classifier: Operating System :: OS Independent
|
|
11
|
-
Requires-Python: >=3.8
|
|
12
|
-
Description-Content-Type: text/markdown
|
|
13
|
-
Requires-Dist: pynput
|
|
14
|
-
Requires-Dist: pillow
|
|
15
|
-
Requires-Dist: google-genai
|
|
File without changes
|
|
File without changes
|
|
File without changes
|