screen-controller-ai 0.1.1__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,59 @@
1
+ Metadata-Version: 2.4
2
+ Name: screen-controller-ai
3
+ Version: 0.1.2
4
+ Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features.
5
+ License: MIT
6
+ Requires-Python: >=3.8
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: google-genai
9
+ Requires-Dist: pillow
10
+ Requires-Dist: pynput
11
+ Requires-Dist: pyautogui
12
+
13
+ # Screen Controller AI
14
+
15
+ A safe, dual-model agentic desktop controller with human-in-the-loop safety features built using Python and the Google GenAI SDK.
16
+
17
+ ## Installation
18
+
19
+ Install the package via pip:
20
+
21
+
22
+ pip install screen-controller-ai
23
+
24
+ Quick Start Example
25
+
26
+ Here is how you use the custom interface, with every single line explained step-by-step:
27
+
28
+ Python
29
+
30
+
31
+
32
+ import screen_controller_ai as x # (imports the screen controller package and renames it to x for easy shorthand use)
33
+
34
+ x.allow_controller() # (enables safety monitoring so any mouse movement or keyboard click by the user instantly halts the controller and returns full control)
35
+
36
+ x.open("[https://www.duckdns.org](https://www.duckdns.org)") # (opens the target website or application URL where the task needs to be performed (you may ofcourse choose other websites))
37
+
38
+ x.key = "YOUR_GEMINI_API_KEY" # (sets the API key so the AI models can connect and run in the background)
39
+
40
+ x.target = "get a free sub domain from duck dns" # (defines the specific goal for the AI to complete while a secondary background model verifies its progress)
41
+
42
+ x.allow_screen_viewer() # (gives the AI permission to view and capture the computer screen live)
43
+
44
+ x.start() # (launches the automated process so you can watch live as the AI works toward your target)
45
+ Features
46
+ Instant Safety Interruption: Automatically stops execution the moment any human mouse movement or keyboard click is detected.
47
+
48
+ Dual-Model Verification: Uses Gemini Flash for task execution and Gemini Flash Lite in the background to verify progress.
49
+
50
+ Simple Python Interface: Clean shorthand syntax designed for quick automation tasks.
51
+
52
+ Requirements
53
+ Python >= 3.8
54
+
55
+ pynput
56
+
57
+ pillow
58
+
59
+ google-genai
@@ -0,0 +1,47 @@
1
+ # Screen Controller AI
2
+
3
+ A safe, dual-model agentic desktop controller with human-in-the-loop safety features built using Python and the Google GenAI SDK.
4
+
5
+ ## Installation
6
+
7
+ Install the package via pip:
8
+
9
+
10
+ pip install screen-controller-ai
11
+
12
+ Quick Start Example
13
+
14
+ Here is how you use the custom interface, with every single line explained step-by-step:
15
+
16
+ Python
17
+
18
+
19
+
20
+ import screen_controller_ai as x # (imports the screen controller package and renames it to x for easy shorthand use)
21
+
22
+ x.allow_controller() # (enables safety monitoring so any mouse movement or keyboard click by the user instantly halts the controller and returns full control)
23
+
24
+ x.open("[https://www.duckdns.org](https://www.duckdns.org)") # (opens the target website or application URL where the task needs to be performed (you may ofcourse choose other websites))
25
+
26
+ x.key = "YOUR_GEMINI_API_KEY" # (sets the API key so the AI models can connect and run in the background)
27
+
28
+ x.target = "get a free sub domain from duck dns" # (defines the specific goal for the AI to complete while a secondary background model verifies its progress)
29
+
30
+ x.allow_screen_viewer() # (gives the AI permission to view and capture the computer screen live)
31
+
32
+ x.start() # (launches the automated process so you can watch live as the AI works toward your target)
33
+ Features
34
+ Instant Safety Interruption: Automatically stops execution the moment any human mouse movement or keyboard click is detected.
35
+
36
+ Dual-Model Verification: Uses Gemini Flash for task execution and Gemini Flash Lite in the background to verify progress.
37
+
38
+ Simple Python Interface: Clean shorthand syntax designed for quick automation tasks.
39
+
40
+ Requirements
41
+ Python >= 3.8
42
+
43
+ pynput
44
+
45
+ pillow
46
+
47
+ google-genai
@@ -0,0 +1,168 @@
1
+ import time
2
+ import threading
3
+ import webbrowser
4
+ from PIL import ImageGrab
5
+ import pyautogui
6
+ from pynput import mouse, keyboard
7
+ from google import genai
8
+
9
+ # Module-level state variables
10
+ key = None
11
+ target = ""
12
+ _controller_allowed = False
13
+ _screen_viewer_allowed = False
14
+ _is_running = False
15
+ _listener_active = False
16
+
17
+ # Fail-safe for Pyautogui: moving mouse to any corner instantly aborts as extra safety
18
+ pyautogui.FAILSAFE = True
19
+
20
+ def allow_controller():
21
+ """Enables safety monitoring so any human mouse movement or keyboard click instantly halts execution."""
22
+ global _controller_allowed, _listener_active
23
+ _controller_allowed = True
24
+ print("[Safety] Controller safety monitor armed. Any mouse/keyboard action will halt execution.")
25
+ if not _listener_active:
26
+ _start_safety_listener()
27
+
28
+ def open(url):
29
+ """Opens the specified URL in the default web browser."""
30
+ print(f"[Action] Opening URL: {url}")
31
+ webbrowser.open(url)
32
+ time.sleep(3) # Give browser time to load
33
+
34
+ def allow_screen_viewer():
35
+ """Grants permission for the AI to capture and analyze the screen."""
36
+ global _screen_viewer_allowed
37
+ _screen_viewer_allowed = True
38
+ print("[Vision] Screen viewer permission granted.")
39
+
40
+ def _start_safety_listener():
41
+ """Fires background listeners to detect human input and abort instantly."""
42
+ global _listener_active, _is_running
43
+ _listener_active = True
44
+
45
+ def on_move(x, y):
46
+ if _is_running:
47
+ print("\n[ALERT] Human mouse movement detected! Halting controller instantly.")
48
+ _stop_execution()
49
+
50
+ def on_click(x, y, button, pressed):
51
+ if _is_running and pressed:
52
+ print("\n[ALERT] Human mouse click detected! Halting controller instantly.")
53
+ _stop_execution()
54
+
55
+ def on_press(key_pressed):
56
+ if _is_running:
57
+ print(f"\n[ALERT] Human keyboard input detected! Halting controller instantly.")
58
+ _stop_execution()
59
+
60
+ mouse_listener = mouse.Listener(on_move=on_move, on_click=on_click)
61
+ keyboard_listener = keyboard.Listener(on_press=on_press)
62
+
63
+ mouse_listener.daemon = True
64
+ keyboard_listener.daemon = True
65
+
66
+ mouse_listener.start()
67
+ keyboard_listener.start()
68
+
69
+ def _stop_execution():
70
+ global _is_running
71
+ _is_running = False
72
+
73
+ def start():
74
+ """Starts the dual-model AI loop to see the screen and control the desktop toward the target."""
75
+ global _is_running, key, target, _controller_allowed, _screen_viewer_allowed
76
+
77
+ if not _controller_allowed:
78
+ raise RuntimeError("Controller not allowed! Call x.allow_controller() first.")
79
+ if not _screen_viewer_allowed:
80
+ raise RuntimeError("Screen viewer not allowed! Call x.allow_screen_viewer() first.")
81
+ if not key:
82
+ raise ValueError("API key not set! Assign x.key.")
83
+ if not target:
84
+ raise ValueError("Target not set! Assign x.target.")
85
+
86
+ _is_running = True
87
+ print(f"\n[Started] Running task target: '{target}'")
88
+
89
+ client = genai.Client(api_key=key if key.startswith("AIza") else None)
90
+
91
+ step_count = 0
92
+ while _is_running and step_count < 10:
93
+ step_count += 1
94
+ print(f"\n--- Execution Step {step_count} ---")
95
+
96
+ # 1. Capture current desktop screen so the AI can SEE it
97
+ screenshot = ImageGrab.grab()
98
+ screenshot_path = f"screenshot_step_{step_count}.png"
99
+ screenshot.save(screenshot_path)
100
+ print(f"[Vision] Screenshot captured.")
101
+
102
+ # 2. Ask Gemini to analyze screen and return action coordinates or text to type
103
+ prompt = f"""
104
+ Goal: {target}
105
+ Look at the attached screen capture. Decide the next action to control the computer.
106
+ Format your response strictly as one of these commands:
107
+ - CLICK X, Y (to click specific pixel coordinates on screen)
108
+ - TYPE "text" (to type text using keyboard)
109
+ - PRESS key (to press a specific key like enter, tab)
110
+ - DONE (if the goal is fully achieved)
111
+ """
112
+
113
+ try:
114
+ # Main model executing the action
115
+ response = client.models.generate_content(
116
+ model='gemini-2.5-flash',
117
+ contents=[prompt, screenshot]
118
+ )
119
+ action_text = response.text.strip()
120
+ print(f"[AI Action]: {action_text}")
121
+
122
+ if "DONE" in action_text.upper():
123
+ print("[Success] AI reports the task is complete!")
124
+ break
125
+
126
+ # Parse and physically EXECUTE the action on the user's screen
127
+ if "CLICK" in action_text.upper():
128
+ # Extract coordinates e.g., CLICK 500, 300
129
+ try:
130
+ coords = action_text.split("CLICK")[1].strip().replace("(", "").replace(")", "").split(",")
131
+ cx, cy = int(coords[0].strip()), int(coords[1].strip())
132
+ print(f"[Control] Clicking at X={cx}, Y={cy}")
133
+ pyautogui.click(cx, cy)
134
+ except Exception as ex:
135
+ print(f"[Control Error] Could not parse click coordinates: {ex}")
136
+
137
+ elif "TYPE" in action_text.upper():
138
+ # Extract text inside quotes
139
+ try:
140
+ text_to_type = action_text.split('"')[1]
141
+ print(f"[Control] Typing: '{text_to_type}'")
142
+ pyautogui.write(text_to_type, interval=0.05)
143
+ except Exception as ex:
144
+ print(f"[Control Error] Could not parse text to type: {ex}")
145
+
146
+ elif "PRESS" in action_text.upper():
147
+ try:
148
+ key_to_press = action_text.split("PRESS")[1].strip().lower()
149
+ print(f"[Control] Pressing key: {key_to_press}")
150
+ pyautogui.press(key_to_press)
151
+ except Exception as ex:
152
+ print(f"[Control Error] Could not parse key: {ex}")
153
+
154
+ # Background verifier model checking progress
155
+ verifier_prompt = f"Verify if the goal '{target}' is successfully progressing based on current desktop action."
156
+ verifier_response = client.models.generate_content(
157
+ model='gemini-2.5-flash-lite',
158
+ contents=[verifier_prompt]
159
+ )
160
+ print(f"[Background Verifier]: {verifier_response.text.strip()}")
161
+
162
+ except Exception as e:
163
+ print(f"[Error]: {e}")
164
+ break
165
+
166
+ time.sleep(3)
167
+
168
+ print("\n[Finished] Screen Controller session completed or halted.")
@@ -0,0 +1,17 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0.0"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "screen-controller-ai"
7
+ version = "0.1.2"
8
+ description = "A safe, dual-model agentic desktop controller with human-in-the-loop safety features."
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = { text = "MIT" }
12
+ dependencies = [
13
+ "google-genai",
14
+ "pillow",
15
+ "pynput",
16
+ "pyautogui"
17
+ ]
@@ -0,0 +1,59 @@
1
+ Metadata-Version: 2.4
2
+ Name: screen-controller-ai
3
+ Version: 0.1.2
4
+ Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features.
5
+ License: MIT
6
+ Requires-Python: >=3.8
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: google-genai
9
+ Requires-Dist: pillow
10
+ Requires-Dist: pynput
11
+ Requires-Dist: pyautogui
12
+
13
+ # Screen Controller AI
14
+
15
+ A safe, dual-model agentic desktop controller with human-in-the-loop safety features built using Python and the Google GenAI SDK.
16
+
17
+ ## Installation
18
+
19
+ Install the package via pip:
20
+
21
+
22
+ pip install screen-controller-ai
23
+
24
+ Quick Start Example
25
+
26
+ Here is how you use the custom interface, with every single line explained step-by-step:
27
+
28
+ Python
29
+
30
+
31
+
32
+ import screen_controller_ai as x # (imports the screen controller package and renames it to x for easy shorthand use)
33
+
34
+ x.allow_controller() # (enables safety monitoring so any mouse movement or keyboard click by the user instantly halts the controller and returns full control)
35
+
36
+ x.open("[https://www.duckdns.org](https://www.duckdns.org)") # (opens the target website or application URL where the task needs to be performed (you may ofcourse choose other websites))
37
+
38
+ x.key = "YOUR_GEMINI_API_KEY" # (sets the API key so the AI models can connect and run in the background)
39
+
40
+ x.target = "get a free sub domain from duck dns" # (defines the specific goal for the AI to complete while a secondary background model verifies its progress)
41
+
42
+ x.allow_screen_viewer() # (gives the AI permission to view and capture the computer screen live)
43
+
44
+ x.start() # (launches the automated process so you can watch live as the AI works toward your target)
45
+ Features
46
+ Instant Safety Interruption: Automatically stops execution the moment any human mouse movement or keyboard click is detected.
47
+
48
+ Dual-Model Verification: Uses Gemini Flash for task execution and Gemini Flash Lite in the background to verify progress.
49
+
50
+ Simple Python Interface: Clean shorthand syntax designed for quick automation tasks.
51
+
52
+ Requirements
53
+ Python >= 3.8
54
+
55
+ pynput
56
+
57
+ pillow
58
+
59
+ google-genai
@@ -1,3 +1,4 @@
1
+ README.md
1
2
  controller.py
2
3
  pyproject.toml
3
4
  screen_controller_ai.egg-info/PKG-INFO
@@ -1,3 +1,4 @@
1
- pynput
2
- pillow
3
1
  google-genai
2
+ pillow
3
+ pynput
4
+ pyautogui
@@ -1,15 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: screen-controller-ai
3
- Version: 0.1.1
4
- Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features
5
- Author: Atharva
6
- License: MIT
7
- Project-URL: Homepage, https://github.com/atharva/screen-controller
8
- Classifier: Programming Language :: Python :: 3
9
- Classifier: License :: OSI Approved :: MIT License
10
- Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.8
12
- Description-Content-Type: text/markdown
13
- Requires-Dist: pynput
14
- Requires-Dist: pillow
15
- Requires-Dist: google-genai
@@ -1,117 +0,0 @@
1
- import time
2
- import threading
3
- import sys
4
- from pynput import mouse, keyboard
5
- from PIL import ImageGrab
6
-
7
- class ScreenController:
8
- def __init__(self):
9
- self.url = None
10
- self.api_key = None
11
- self.task_target = None
12
- self.screen_viewer_allowed = False
13
- self.controller_allowed = False
14
- self._stop_event = threading.Event()
15
-
16
- self._mouse_listener = None
17
- self._keyboard_listener = None
18
-
19
- def allow_controller(self):
20
- """Enables the safety interrupt listener (mouse/keyboard tripwire)."""
21
- self.controller_allowed = True
22
- print("[Safety] Controller armed. Move your mouse or press any key to instantly abort.")
23
- self._start_safety_listeners()
24
- return self
25
-
26
- def open(self, url):
27
- """Sets the target URL to open."""
28
- self.url = url
29
- print(f"[Action] Target URL configured: {self.url}")
30
- return self
31
-
32
- def key(self, api_key):
33
- """Sets the API key for the AI models."""
34
- self.api_key = api_key
35
- print("[Config] API Key loaded successfully.")
36
- return self
37
-
38
- def target(self, goal):
39
- """Defines what the AI agent must accomplish."""
40
- self.task_target = goal
41
- print(f"[Target] Goal defined: '{self.task_target}'")
42
- return self
43
-
44
- def allow_screen_viewer(self):
45
- """Grants permission for the model to capture screenshots."""
46
- self.screen_viewer_allowed = True
47
- print("[Permission] Screen viewer enabled.")
48
- return self
49
-
50
- def _trigger_abort(self, reason="User input detected"):
51
- """Instantly stops the execution loop."""
52
- if not self._stop_event.is_set():
53
- print(f"\n[ABORTED] Safety tripwire triggered ({reason}). Stopping controller immediately.")
54
- self._stop_event.set()
55
- self._stop_safety_listeners()
56
-
57
- def _start_safety_listeners(self):
58
- """Starts background listeners to detect user mouse movement or key presses."""
59
- def on_move(x, y):
60
- self._trigger_abort("Mouse moved")
61
-
62
- def on_click(x, y, button, pressed):
63
- self._trigger_abort("Mouse clicked")
64
-
65
- def on_press(key):
66
- self._trigger_abort(f"Key pressed: {key}")
67
-
68
- # Initialize non-blocking listeners
69
- self._mouse_listener = mouse.Listener(on_move=on_move, on_click=on_click)
70
- self._keyboard_listener = keyboard.Listener(on_press=on_press)
71
-
72
- self._mouse_listener.start()
73
- self._keyboard_listener.start()
74
-
75
- def _stop_safety_listeners(self):
76
- """Cleanly stops the safety listeners."""
77
- if self._mouse_listener:
78
- self._mouse_listener.stop()
79
- if self._keyboard_listener:
80
- self._keyboard_listener.stop()
81
-
82
- def start(self):
83
- """Executes the dual-model processing and action loop."""
84
- if not self.controller_allowed or not self.screen_viewer_allowed:
85
- raise PermissionError("You must allow controller and screen viewer before starting!")
86
-
87
- if not self.api_key:
88
- raise ValueError("API key is missing! Set it using x.key().")
89
-
90
- print(f"\n[Running] Starting task: '{self.task_target}'")
91
- print("[Model] Main execution model active.")
92
- print("[Model] Background verification model active.")
93
-
94
- try:
95
- while not self._stop_event.is_set():
96
- # 1. Capture screen snapshot safely
97
- if self.screen_viewer_allowed:
98
- screenshot = ImageGrab.grab()
99
- # (Here is where you would send the screenshot to Gemini Flash Lite)
100
-
101
- # 2. Main Model execution step & Background Verifier check
102
- print("[Looping] Analyzing screen state and verifying task progress...")
103
-
104
- # Simulate a working interval (check every 2 seconds)
105
- for _ in range(20):
106
- if self._stop_event.is_set():
107
- break
108
- time.sleep(0.1)
109
-
110
- # Break condition placeholder for testing
111
- break
112
-
113
- except Exception as e:
114
- print(f"[Error] {e}")
115
- finally:
116
- self._stop_safety_listeners()
117
- print("[Stopped] Screen controller session closed safely.")
@@ -1,27 +0,0 @@
1
- [build-system]
2
- requires = ["setuptools>=61.0.0"]
3
- build-backend = "setuptools.build_meta"
4
-
5
- [project]
6
- name = "screen-controller-ai"
7
- version = "0.1.1"
8
- description = "A safe, dual-model agentic desktop controller with human-in-the-loop safety features"
9
- readme = "README.md"
10
- authors = [
11
- { name = "Atharva" }
12
- ]
13
- license = { text = "MIT" }
14
- classifiers = [
15
- "Programming Language :: Python :: 3",
16
- "License :: OSI Approved :: MIT License",
17
- "Operating System :: OS Independent",
18
- ]
19
- requires-python = ">=3.8"
20
- dependencies = [
21
- "pynput",
22
- "pillow",
23
- "google-genai"
24
- ]
25
-
26
- [project.urls]
27
- Homepage = "https://github.com/atharva/screen-controller"
@@ -1,15 +0,0 @@
1
- Metadata-Version: 2.4
2
- Name: screen-controller-ai
3
- Version: 0.1.1
4
- Summary: A safe, dual-model agentic desktop controller with human-in-the-loop safety features
5
- Author: Atharva
6
- License: MIT
7
- Project-URL: Homepage, https://github.com/atharva/screen-controller
8
- Classifier: Programming Language :: Python :: 3
9
- Classifier: License :: OSI Approved :: MIT License
10
- Classifier: Operating System :: OS Independent
11
- Requires-Python: >=3.8
12
- Description-Content-Type: text/markdown
13
- Requires-Dist: pynput
14
- Requires-Dist: pillow
15
- Requires-Dist: google-genai