windows-mcp 0.3.0__py3-none-any.whl

desktop/__init__.py

@@ -0,0 +1,247 @@
+from uiautomation import Control, GetRootControl, ControlType, GetFocusedControl, ShowWindow, IsTopLevelWindow, IsZoomed, IsIconic, IsWindowVisible, ControlFromHandle
+from src.desktop.config import EXCLUDED_CLASSNAMES,BROWSER_NAMES
+from src.desktop.views import DesktopState,App,Size
+from fuzzywuzzy import process
+from psutil import Process
+from src.tree import Tree
+from time import sleep
+import pyautogui as pg
+from io import BytesIO
+from PIL import Image
+import subprocess
+import ctypes
+import csv
+import os
+import io
+
+class Desktop:
+    def __init__(self):
+        ctypes.windll.user32.SetProcessDPIAware()
+        self.desktop_state=None
+        
+    def get_state(self,use_vision:bool=False)->DesktopState:
+        tree=Tree(self)
+        tree_state=tree.get_state()
+        if use_vision:
+            nodes=tree_state.interactive_nodes
+            annotated_screenshot=tree.annotated_screenshot(nodes=nodes,scale=0.5)
+            screenshot=self.screenshot_in_bytes(screenshot=annotated_screenshot)
+        else:
+            screenshot=None
+        active_app,apps=self.get_apps()
+        self.desktop_state=DesktopState(apps=apps,active_app=active_app,screenshot=screenshot,tree_state=tree_state)
+        return self.desktop_state
+    
+    def get_active_app(self,apps:list[App])->App|None:
+        if len(apps)>0 and apps[0].status != "Minimized":
+            return apps[0]
+        return None
+    
+    def get_app_status(self,control:Control)->str:
+        if IsIconic(control.NativeWindowHandle):
+            return 'Minimized'
+        elif IsZoomed(control.NativeWindowHandle):
+            return 'Maximized'
+        elif IsWindowVisible(control.NativeWindowHandle):
+            return 'Normal'
+        else:
+            return 'Hidden'
+    
+    def get_window_element_from_element(self,element:Control)->Control|None:
+        while element is not None:
+            if IsTopLevelWindow(element.NativeWindowHandle):
+                return element
+            element = element.GetParentControl()
+        return None
+
+    def get_element_under_cursor(self)->Control:
+        return GetFocusedControl()
+    
+    def get_default_browser(self):
+        mapping = {
+            "ChromeHTML": "Google Chrome",
+            "FirefoxURL": "Mozilla Firefox",
+            "MSEdgeHTM": "Microsoft Edge",
+            "IE.HTTP": "Internet Explorer",
+            "OperaStable": "Opera",
+            "BraveHTML": "Brave",
+            "SafariHTML": "Safari"
+        }
+        command= "(Get-ItemProperty HKCU:\\Software\\Microsoft\\Windows\\Shell\\Associations\\UrlAssociations\\http\\UserChoice).ProgId"
+        browser,_=self.execute_command(command)
+        return mapping.get(browser.strip())
+        
+    def get_default_language(self)->str:
+        command="Get-Culture | Select-Object Name,DisplayName | ConvertTo-Csv -NoTypeInformation"
+        response,_=self.execute_command(command)
+        reader=csv.DictReader(io.StringIO(response))
+        return "".join([row.get('DisplayName') for row in reader])
+    
+    def get_apps_from_start_menu(self)->dict[str,str]:
+        command='Get-StartApps | ConvertTo-Csv -NoTypeInformation'
+        apps_info,_=self.execute_command(command)
+        reader=csv.DictReader(io.StringIO(apps_info))
+        return {row.get('Name').lower():row.get('AppID') for row in reader}
+    
+    def execute_command(self,command:str)->tuple[str,int]:
+        try:
+            # Use UTF-8 encoding for better Chinese character support
+            result = subprocess.run(
+                ['powershell', '-NoProfile', '-Command', 
+                 '[Console]::OutputEncoding = [System.Text.Encoding]::UTF8; ' + command], 
+                capture_output=True, check=True, text=True, encoding='utf-8',cwd=os.path.expanduser(path='~\\Desktop')
+            )
+            return (result.stdout, result.returncode)
+        except subprocess.CalledProcessError as e:
+            try:
+                # Try UTF-8 first
+                error_output = e.stdout if hasattr(e, 'stdout') and e.stdout else ''
+                return (error_output, e.returncode)
+            except Exception:
+                # Fallback to GBK for Chinese Windows systems
+                try:
+                    result = subprocess.run(
+                        ['powershell', '-NoProfile', '-Command', command], 
+                        capture_output=True, check=False
+                    )
+                    return (result.stdout.decode('gbk', errors='ignore'), result.returncode)
+                except Exception:
+                    return ('Command execution failed with encoding issues', 1)
+        
+    def is_app_browser(self,node:Control):
+        process=Process(node.ProcessId)
+        return process.name() in BROWSER_NAMES
+    
+    def get_windows_version(self)->str:
+        response,status=self.execute_command("(Get-CimInstance Win32_OperatingSystem).Caption")
+        if status==0:
+            return response.strip()
+        return "Windows"
+    
+    def resize_app(self,size:tuple[int,int]=None,loc:tuple[int,int]=None)->tuple[str,int]:
+        active_app=self.desktop_state.active_app
+        if active_app is None:
+            return ('No active app found',1)
+        app_control=ControlFromHandle(active_app.handle)
+        if loc is None:
+            x=app_control.BoundingRectangle.left
+            y=app_control.BoundingRectangle.top
+            loc=(x,y)
+        if size is None:
+            width=app_control.BoundingRectangle.width()
+            height=app_control.BoundingRectangle.height()
+            size=(width,height)
+        x,y=loc
+        width,height=size
+        app_control.MoveWindow(x,y,width,height)
+        return (f'Application {active_app.name.title()} resized to {width}x{height} at {x},{y}.',0)
+        
+    def launch_app(self,name:str)->tuple[str,int]:
+        apps_map=self.get_apps_from_start_menu()
+        
+        # Improved fuzzy matching for Chinese and English app names
+        # First try exact match (case insensitive)
+        exact_matches = {k: v for k, v in apps_map.items() if name.lower() in k.lower() or k.lower() in name.lower()}
+        if exact_matches:
+            # Use the first exact match
+            app_name = list(exact_matches.keys())[0]
+            app_id = exact_matches[app_name]
+            if app_id.endswith('.exe'):
+                _,status=self.execute_command(f'Start-Process "{app_id}"')
+            else:
+                _,status=self.execute_command(f'Start-Process "shell:AppsFolder\\{app_id}"')
+            response=f'Launched {name.title()}. Wait for the app to launch...'
+            return response,status
+        
+        # If no exact match, use fuzzy matching with lower threshold for Chinese
+        matched_app=process.extractOne(name,apps_map,score_cutoff=60)
+        if matched_app is not None:
+            app_id,_,app_name=matched_app
+            if app_id.endswith('.exe'):
+                _,status=self.execute_command(f'Start-Process "{app_id}"')
+            else:
+                _,status=self.execute_command(f'Start-Process "shell:AppsFolder\\{app_id}"')
+            response=f'Launched {name.title()}. Wait for the app to launch...'
+            return response,status
+        
+        # Try partial matching for Chinese characters
+        for app_name, app_id in apps_map.items():
+            if any(char in app_name for char in name) or any(char in name for char in app_name):
+                if app_id.endswith('.exe'):
+                    _,status=self.execute_command(f'Start-Process "{app_id}"')
+                else:
+                    _,status=self.execute_command(f'Start-Process "shell:AppsFolder\\{app_id}"')
+                response=f'Launched {name.title()}. Wait for the app to launch...'
+                return response,status
+        
+        return (f'Application {name.title()} not found in start menu. Available apps with similar names: {list(apps_map.keys())[:5]}',1)
+    
+    def switch_app(self,name:str)->tuple[str,int]:
+        apps={app.name:app for app in [self.desktop_state.active_app]+self.desktop_state.apps if app is not None}
+        matched_app:tuple[str,float]=process.extractOne(name,list(apps.keys()))
+        if matched_app is None:
+            return (f'Application {name.title()} not found.',1)
+        app_name,_=matched_app
+        app=apps.get(app_name)
+        if IsIconic(app.handle):
+            ShowWindow(app.handle, cmdShow=9)
+            return (f'{app_name.title()} restored from minimized state.',0)
+        else:
+            shortcut=['alt','tab']
+            for app in apps.values():
+                if app.name==app_name:
+                    break
+                pg.hotkey(*shortcut)
+                pg.sleep(0.1)
+            return (f'Switched to {app_name.title()} window.',0)
+
+    def get_app_size(self,control:Control):
+        window=control.BoundingRectangle
+        if window.isempty():
+            return Size(width=0,height=0)
+        return Size(width=window.width(),height=window.height())
+    
+    def is_app_visible(self,app)->bool:
+        is_minimized=self.get_app_status(app)!='Minimized'
+        size=self.get_app_size(app)
+        area=size.width*size.height
+        is_overlay=self.is_overlay_app(app)
+        return not is_overlay and is_minimized and area>10
+    
+    def is_overlay_app(self,element:Control) -> bool:
+        no_children = len(element.GetChildren()) == 0
+        is_name = "Overlay" in element.Name.strip()
+        return no_children or is_name
+        
+    def get_apps(self) -> tuple[App|None,list[App]]:
+        try:
+            sleep(0.5)
+            desktop = GetRootControl()  # Get the desktop control
+            elements = desktop.GetChildren()
+            apps = []
+            for depth, element in enumerate(elements):
+                if element.ClassName in EXCLUDED_CLASSNAMES or self.is_overlay_app(element):
+                    continue
+                if element.ControlType in [ControlType.WindowControl, ControlType.PaneControl]:
+                    status = self.get_app_status(element)
+                    size=self.get_app_size(element)
+                    apps.append(App(name=element.Name, depth=depth, status=status, size=size, process_id=element.ProcessId, handle=element.NativeWindowHandle))
+        except Exception as ex:
+            print(f"Error: {ex}")
+            apps = []
+
+        active_app=self.get_active_app(apps)
+        apps=apps[1:] if len(apps)>1 else []
+        return (active_app,apps)
+    
+    def screenshot_in_bytes(self,screenshot:Image.Image)->bytes:
+        io=BytesIO()
+        screenshot.save(io,format='PNG')
+        bytes=io.getvalue()
+        return bytes
+
+    def get_screenshot(self,scale:float=0.7)->Image.Image:
+        screenshot=pg.screenshot()
+        size=(screenshot.width*scale, screenshot.height*scale)
+        screenshot.thumbnail(size=size, resample=Image.Resampling.LANCZOS)
+        return screenshot

desktop/config.py

@@ -0,0 +1,11 @@
+from typing import Set
+
+BROWSER_NAMES=set(['msedge.exe','chrome.exe','firefox.exe'])
+
+AVOIDED_APPS:Set[str]=set([
+    'Recording toolbar'
+])
+
+EXCLUDED_CLASSNAMES:Set[str]=set([
+    'Progman','Shell_TrayWnd','Microsoft.UI.Content.PopupWindowSiteBridge','Windows.UI.Core.CoreWindow'
+])

desktop/views.py

@@ -0,0 +1,40 @@
+from src.tree.views import TreeState
+from dataclasses import dataclass
+from typing import Literal,Optional
+
+@dataclass
+class App:
+    name:str
+    depth:int
+    status:Literal['Maximized','Minimized','Normal']
+    size:'Size'
+    process_id:int
+    handle:int
+
+    def to_string(self):
+        return f'Name: {self.name} Depth: {self.depth} Status: {self.status} Size: {self.size.to_string()}'
+
+@dataclass
+class Size:
+    width:int
+    height:int
+
+    def to_string(self):
+        return f'({self.width},{self.height})'
+
+@dataclass
+class DesktopState:
+    apps:list[App]
+    active_app:Optional[App]
+    screenshot:bytes|None
+    tree_state:TreeState
+
+    def active_app_to_string(self):
+        if self.active_app is None:
+            return 'No active app'
+        return self.active_app.to_string()
+
+    def apps_to_string(self):
+        if len(self.apps)==0:
+            return 'No apps opened'
+        return '\n'.join([app.to_string() for app in self.apps])

tree/__init__.py

@@ -0,0 +1,303 @@
+from src.tree.config import INTERACTIVE_CONTROL_TYPE_NAMES,INFORMATIVE_CONTROL_TYPE_NAMES, DEFAULT_ACTIONS, THREAD_MAX_RETRIES
+from src.tree.views import TreeElementNode, TextElementNode, ScrollElementNode, Center, BoundingBox, TreeState
+from uiautomation import GetRootControl,Control,ImageControl,ScrollPattern
+from src.tree.utils import random_point_within_bounding_box
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from src.desktop.config import AVOIDED_APPS, EXCLUDED_CLASSNAMES
+from PIL import Image, ImageFont, ImageDraw
+from typing import TYPE_CHECKING
+from time import sleep
+import random
+
+if TYPE_CHECKING:
+    from src.desktop import Desktop
+
+class Tree:
+    def __init__(self,desktop:'Desktop'):
+        self.desktop=desktop
+
+    def get_state(self)->TreeState:
+        sleep(0.5)
+        # Get the root control of the desktop
+        root=GetRootControl()
+        interactive_nodes,informative_nodes,scrollable_nodes=self.get_appwise_nodes(node=root)
+        return TreeState(interactive_nodes=interactive_nodes,informative_nodes=informative_nodes,scrollable_nodes=scrollable_nodes)
+    
+    def get_appwise_nodes(self,node:Control) -> tuple[list[TreeElementNode],list[TextElementNode]]:
+        apps:list[Control]=[]
+        found_foreground_app=False
+
+        for app in node.GetChildren():
+            if app.ClassName in EXCLUDED_CLASSNAMES:
+                apps.append(app)
+            elif app.Name not in AVOIDED_APPS and self.desktop.is_app_visible(app):
+                if not found_foreground_app:
+                    apps.append(app)
+                    found_foreground_app=True
+
+        interactive_nodes,informative_nodes,scrollable_nodes=[],[],[]
+        # Parallel traversal (using ThreadPoolExecutor) to get nodes from each app
+        with ThreadPoolExecutor() as executor:
+            retry_counts = {app: 0 for app in apps}
+            future_to_app = {executor.submit(self.get_nodes, app, self.desktop.is_app_browser(app)): app for app in apps}
+            while future_to_app:  # keep running until no pending futures
+                for future in as_completed(list(future_to_app)):
+                    app = future_to_app.pop(future)  # remove completed future
+                    try:
+                        result = future.result()
+                        if result:
+                            element_nodes, text_nodes, scroll_nodes = result
+                            interactive_nodes.extend(element_nodes)
+                            informative_nodes.extend(text_nodes)
+                            scrollable_nodes.extend(scroll_nodes)
+                    except Exception as e:
+                        retry_counts[app] += 1
+                        print(f"Error in processing node {app.Name}, retry attempt {retry_counts[app]}\nError: {e}")
+                        if retry_counts[app] < THREAD_MAX_RETRIES:
+                            new_future = executor.submit(self.get_nodes, app, self.desktop.is_app_browser(app))
+                            future_to_app[new_future] = app
+                        else:
+                            print(f"Task failed completely for {app.Name} after {THREAD_MAX_RETRIES} retries")
+        return interactive_nodes,informative_nodes,scrollable_nodes
+
+    def get_nodes(self, node: Control, is_browser=False) -> tuple[list[TreeElementNode],list[TextElementNode],list[ScrollElementNode]]:
+        interactive_nodes, informative_nodes, scrollable_nodes = [], [], []
+        app_name=node.Name.strip()
+        app_name='Desktop' if node.ClassName=='Progman' else app_name
+        
+        def is_element_visible(node:Control,threshold:int=0):
+            is_control=node.IsControlElement
+            box=node.BoundingRectangle
+            if box.isempty():
+                return False
+            width=box.width()
+            height=box.height()
+            area=width*height
+            is_offscreen=(not node.IsOffscreen) or node.ControlTypeName in ['EditControl']
+            return area > threshold and is_offscreen and is_control
+    
+        def is_element_enabled(node:Control):
+            try:
+                return node.IsEnabled
+            except Exception:
+                return False
+            
+        def is_default_action(node:Control):
+            legacy_pattern=node.GetLegacyIAccessiblePattern()
+            default_action=legacy_pattern.DefaultAction.title()
+            if default_action in DEFAULT_ACTIONS:
+                return True
+            return False
+        
+        def is_element_image(node:Control):
+            if isinstance(node,ImageControl):
+                if node.LocalizedControlType=='graphic' or not node.IsKeyboardFocusable:
+                    return True
+            return False
+        
+        def is_element_text(node:Control):
+            try:
+                if node.ControlTypeName in INFORMATIVE_CONTROL_TYPE_NAMES:
+                    if is_element_visible(node) and is_element_enabled(node) and not is_element_image(node):
+                        return True
+            except Exception:
+                return False
+            return False
+        
+        def is_element_scrollable(node:Control):
+            try:
+                scroll_pattern:ScrollPattern=node.GetScrollPattern()
+                return scroll_pattern.VerticallyScrollable or scroll_pattern.HorizontallyScrollable
+            except Exception:
+                return False
+            
+        def is_keyboard_focusable(node:Control):
+            try:
+                if node.ControlTypeName in set(['EditControl','ButtonControl','CheckBoxControl','RadioButtonControl','TabItemControl']):
+                    return True
+                return node.IsKeyboardFocusable
+            except Exception:
+                return False
+            
+        def element_has_child_element(node:Control,control_type:str,child_control_type:str):
+            if node.LocalizedControlType==control_type:
+                first_child=node.GetFirstChildControl()
+                if first_child is None:
+                    return False
+                return first_child.LocalizedControlType==child_control_type
+            
+        def group_has_no_name(node:Control):
+            try:
+                if node.ControlTypeName=='GroupControl':
+                    if not node.Name.strip():
+                        return True
+                return False
+            except Exception:
+                return False
+            
+        def is_element_interactive(node:Control):
+            try:
+                if node.ControlTypeName in INTERACTIVE_CONTROL_TYPE_NAMES:
+                    if is_element_visible(node) and is_element_enabled(node) and (not is_element_image(node) or is_keyboard_focusable(node)):
+                        return True
+                elif node.ControlTypeName=='GroupControl' and is_browser:
+                    if is_element_visible(node) and is_element_enabled(node) and (is_default_action(node) or is_keyboard_focusable(node)):
+                        return True
+                # elif node.ControlTypeName=='GroupControl' and not is_browser:
+                #     if is_element_visible and is_element_enabled(node) and is_default_action(node):
+                #         return True
+            except Exception:
+                return False
+            return False
+        
+        def dom_correction(node:Control):
+            if element_has_child_element(node,'list item','link') or element_has_child_element(node,'item','link'):
+                interactive_nodes.pop()
+                return None
+            elif group_has_no_name(node):
+                interactive_nodes.pop()
+                if is_keyboard_focusable(node):
+                    child=node
+                    try:
+                        while child.GetFirstChildControl() is not None:
+                            child=child.GetFirstChildControl()
+                    except Exception:
+                        return None
+                    if child.ControlTypeName!='TextControl':
+                        return None
+                    control_type='Edit'
+                    box = node.BoundingRectangle
+                    x,y=box.xcenter(),box.ycenter()
+                    center = Center(x=x,y=y)
+                    interactive_nodes.append(TreeElementNode(
+                        name=child.Name.strip() or "''",
+                        control_type=control_type,
+                        shortcut=node.AcceleratorKey or "''",
+                        bounding_box=BoundingBox(left=box.left,top=box.top,right=box.right,bottom=box.bottom,width=box.width(),height=box.height()),
+                        center=center,
+                        app_name=app_name
+                    ))
+            elif element_has_child_element(node,'link','heading'):
+                interactive_nodes.pop()
+                node=node.GetFirstChildControl()
+                control_type='link'
+                box = node.BoundingRectangle
+                x,y=box.xcenter(),box.ycenter()
+                center = Center(x=x,y=y)
+                interactive_nodes.append(TreeElementNode(
+                    name=node.Name.strip() or "''",
+                    control_type=control_type,
+                    shortcut=node.AcceleratorKey or "''",
+                    bounding_box=BoundingBox(left=box.left,top=box.top,right=box.right,bottom=box.bottom,width=box.width(),height=box.height()),
+                    center=center,
+                    app_name=app_name
+                ))
+            
+        def tree_traversal(node: Control):
+            # Checks to skip the nodes that are not interactive
+            if node.IsOffscreen and (node.ControlTypeName not in set(["EditControl","TitleBarControl"])) and node.ClassName not in set(["Popup","Windows.UI.Core.CoreComponentInputSource"]):
+                return None
+            
+            if is_element_interactive(node):
+                box = node.BoundingRectangle
+                x,y=random_point_within_bounding_box(node=node,scale_factor=0.8)
+                center = Center(x=x,y=y)
+                interactive_nodes.append(TreeElementNode(
+                    name=node.Name.strip() or "''",
+                    control_type=node.LocalizedControlType.title(),
+                    shortcut=node.AcceleratorKey or "''",
+                    bounding_box=BoundingBox(left=box.left,top=box.top,right=box.right,bottom=box.bottom,width=box.width(),height=box.height()),
+                    center=center,
+                    app_name=app_name
+                ))
+                if is_browser:
+                    dom_correction(node)
+            elif is_element_text(node):
+                informative_nodes.append(TextElementNode(
+                    name=node.Name.strip() or "''",
+                    app_name=app_name
+                ))
+            elif is_element_scrollable(node):
+                scroll_pattern:ScrollPattern=node.GetScrollPattern()
+                box = node.BoundingRectangle
+                # Get the center
+                x,y=random_point_within_bounding_box(node=node,scale_factor=0.8)
+                center = Center(x=x,y=y)
+                scrollable_nodes.append(ScrollElementNode(
+                    name=node.Name.strip() or node.LocalizedControlType.capitalize() or "''",
+                    app_name=app_name,
+                    control_type=node.LocalizedControlType.title(),
+                    bounding_box=BoundingBox(left=box.left,top=box.top,right=box.right,bottom=box.bottom,width=box.width(),height=box.height()),
+                    center=center,
+                    horizontal_scrollable=scroll_pattern.HorizontallyScrollable,
+                    vertical_scrollable=scroll_pattern.VerticallyScrollable
+                ))
+            # Recursively check all children
+            for child in node.GetChildren():
+                tree_traversal(child)
+
+        tree_traversal(node)
+        return (interactive_nodes,informative_nodes,scrollable_nodes)
+    
+    def get_random_color(self):
+        return "#{:06x}".format(random.randint(0, 0xFFFFFF))
+
+    def annotated_screenshot(self, nodes: list[TreeElementNode],scale:float=0.7) -> Image.Image:
+        screenshot = self.desktop.get_screenshot(scale=scale)
+        sleep(0.25)
+        # Add padding
+        padding = 20
+        width = screenshot.width + (2 * padding)
+        height = screenshot.height + (2 * padding)
+        padded_screenshot = Image.new("RGB", (width, height), color=(255, 255, 255))
+        padded_screenshot.paste(screenshot, (padding, padding))
+
+        draw = ImageDraw.Draw(padded_screenshot)
+        font_size = 12
+        try:
+            font = ImageFont.truetype('arial.ttf', font_size)
+        except IOError:
+            font = ImageFont.load_default()
+
+        def get_random_color():
+            return "#{:06x}".format(random.randint(0, 0xFFFFFF))
+
+        def draw_annotation(label, node: TreeElementNode):
+            box = node.bounding_box
+            color = get_random_color()
+
+            # Scale and pad the bounding box also clip the bounding box
+            adjusted_box = (
+                int(box.left * scale) + padding,
+                int(box.top * scale) + padding,
+                int(box.right * scale) + padding,
+                int(box.bottom * scale) + padding
+            )
+            # Draw bounding box
+            draw.rectangle(adjusted_box, outline=color, width=2)
+
+            # Label dimensions
+            label_width = draw.textlength(str(label), font=font)
+            label_height = font_size
+            left, top, right, bottom = adjusted_box
+
+            # Label position above bounding box
+            label_x1 = right - label_width
+            label_y1 = top - label_height - 4
+            label_x2 = label_x1 + label_width
+            label_y2 = label_y1 + label_height + 4
+
+            # Draw label background and text
+            draw.rectangle([(label_x1, label_y1), (label_x2, label_y2)], fill=color)
+            draw.text((label_x1 + 2, label_y1 + 2), str(label), fill=(255, 255, 255), font=font)
+
+        # Draw annotations in parallel
+        with ThreadPoolExecutor() as executor:
+            executor.map(draw_annotation, range(len(nodes)), nodes)
+        return padded_screenshot
+    
+    def get_annotated_image_data(self)->tuple[Image.Image,list[TreeElementNode]]:
+        node=GetRootControl()
+        nodes,_,_=self.get_appwise_nodes(node=node)
+        screenshot=self.annotated_screenshot(nodes=nodes,scale=1.0)
+        return screenshot,nodes

tree/config.py

@@ -0,0 +1,17 @@
+INTERACTIVE_CONTROL_TYPE_NAMES=set([
+    'ButtonControl','ListItemControl','MenuItemControl','DocumentControl',
+    'EditControl','CheckBoxControl', 'RadioButtonControl','ComboBoxControl',
+    'HyperlinkControl','SplitButtonControl','TabItemControl',
+    'TreeItemControl','DataItemControl','HeaderItemControl','TextBoxControl',
+    'ImageControl','SpinnerControl','ScrollBarControl'
+])
+
+DEFAULT_ACTIONS=set([
+    'Click','Press','Jump','Check','Uncheck','Double Click'
+])
+
+INFORMATIVE_CONTROL_TYPE_NAMES=set([
+    'TextControl','ImageControl'
+])
+
+THREAD_MAX_RETRIES = 3

tree/utils.py

@@ -0,0 +1,22 @@
+import random
+from uiautomation import Control
+
+def random_point_within_bounding_box(node: Control, scale_factor: float = 1.0) -> tuple[int, int]:
+    """
+    Generate a random point within a scaled-down bounding box.
+
+    Args:
+        node (Control): The node with a bounding rectangle
+        scale_factor (float, optional): The factor to scale down the bounding box. Defaults to 1.0.
+
+    Returns:
+        tuple: A random point (x, y) within the scaled-down bounding box
+    """
+    box = node.BoundingRectangle
+    scaled_width = int(box.width() * scale_factor)
+    scaled_height = int(box.height() * scale_factor)
+    scaled_left = box.left + (box.width() - scaled_width) // 2
+    scaled_top = box.top + (box.height() - scaled_height) // 2
+    x = random.randint(scaled_left, scaled_left + scaled_width)
+    y = random.randint(scaled_top, scaled_top + scaled_height)
+    return (x, y)

tree/views.py

@@ -0,0 +1,70 @@
+from dataclasses import dataclass,field
+
+@dataclass
+class TreeState:
+    interactive_nodes:list['TreeElementNode']=field(default_factory=list)
+    informative_nodes:list['TextElementNode']=field(default_factory=list)
+    scrollable_nodes:list['ScrollElementNode']=field(default_factory=list)
+
+    def interactive_elements_to_string(self)->str:
+        return '\n'.join([f'Label: {index} App Name: {node.app_name} ControlType: {f'{node.control_type} Control'} Name: {node.name} Shortcut: {node.shortcut} Cordinates: {node.center.to_string()}' for index,node in enumerate(self.interactive_nodes)])
+    
+    def informative_elements_to_string(self)->str:
+        return '\n'.join([f'App Name: {node.app_name} Name: {node.name}' for node in self.informative_nodes])
+    
+    def scrollable_elements_to_string(self)->str:
+        n=len(self.interactive_nodes)
+        return '\n'.join([f'Label: {n+index} App Name: {node.app_name} ControlType: {f'{node.control_type} Control'} Name: {node.name} Cordinates: {node.center.to_string()} Horizontal Scrollable: {node.horizontal_scrollable} Vertical Scrollable: {node.vertical_scrollable}' for index,node in enumerate(self.scrollable_nodes)])
+    
+@dataclass
+class BoundingBox:
+    left:int
+    top:int
+    right:int
+    bottom:int
+    width:int
+    height:int
+
+    def xywh_to_string(self):
+        return f'({self.left},{self.top},{self.width},{self.height})'
+    
+    def xyxy_to_string(self):
+        x1,y1,x2,y2=self.convert_xywh_to_xyxy()
+        return f'({x1},{y1},{x2},{y2})'
+    
+    def convert_xywh_to_xyxy(self)->tuple[int,int,int,int]:
+        x1,y1=self.left,self.top
+        x2,y2=self.left+self.width,self.top+self.height
+        return x1,y1,x2,y2
+
+@dataclass
+class Center:
+    x:int
+    y:int
+
+    def to_string(self)->str:
+        return f'({self.x},{self.y})'
+
+@dataclass
+class TreeElementNode:
+    name:str
+    control_type:str
+    shortcut:str
+    bounding_box:BoundingBox
+    center:Center
+    app_name:str
+
+@dataclass
+class TextElementNode:
+    name:str
+    app_name:str
+
+@dataclass
+class ScrollElementNode:
+    name:str
+    control_type:str
+    app_name:str
+    bounding_box:BoundingBox
+    center:Center
+    horizontal_scrollable:bool
+    vertical_scrollable:bool

windows_mcp-0.3.0.dist-info/METADATA

@@ -0,0 +1,361 @@
+Metadata-Version: 2.4
+Name: windows-mcp
+Version: 0.3.0
+Summary: Lightweight MCP Server for interacting with Windows Operating System.
+Author-email: Jeomon George <jeogeoalukka@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 JEOMON GEORGE
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Project-URL: homepage, https://github.com/CursorTouch
+Keywords: windows,mcp,ai,desktop,ai agent
+Requires-Python: >=3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE.md
+Requires-Dist: click>=8.2.1
+Requires-Dist: fastmcp>=2.8.1
+Requires-Dist: fuzzywuzzy>=0.18.0
+Requires-Dist: humancursor>=1.1.5
+Requires-Dist: ipykernel>=6.30.0
+Requires-Dist: live-inspect>=0.1.1
+Requires-Dist: markdownify>=1.1.0
+Requires-Dist: pillow>=11.2.1
+Requires-Dist: psutil>=7.0.0
+Requires-Dist: pyautogui>=0.9.54
+Requires-Dist: pygetwindow>=0.0.9
+Requires-Dist: python-levenshtein>=0.27.1
+Requires-Dist: requests>=2.32.3
+Requires-Dist: uiautomation>=2.0.24
+Dynamic: license-file
+
+<div align="center">
+  <h1>🪟 Windows-MCP</h1>
+
+  <a href="https://github.com/CursorTouch/Windows-MCP/blob/main/LICENSE">
+    <img src="https://img.shields.io/badge/license-MIT-green" alt="License">
+  </a>
+  <img src="https://img.shields.io/badge/python-3.13%2B-blue" alt="Python">
+  <img src="https://img.shields.io/badge/platform-Windows%207–11-blue" alt="Platform: Windows 7 to 11">
+  <img src="https://img.shields.io/github/last-commit/CursorTouch/Windows-MCP" alt="Last Commit">
+  <br>
+  <a href="https://x.com/CursorTouch">
+    <img src="https://img.shields.io/badge/follow-%40CursorTouch-1DA1F2?logo=twitter&style=flat" alt="Follow on Twitter">
+  </a>
+  <a href="https://discord.com/invite/Aue9Yj2VzS">
+    <img src="https://img.shields.io/badge/Join%20on-Discord-5865F2?logo=discord&logoColor=white&style=flat" alt="Join us on Discord">
+  </a>
+
+</div>
+
+<br>
+
+**Windows MCP** is a lightweight, open-source project that enables seamless integration between AI agents and the Windows operating system. Acting as an MCP server bridges the gap between LLMs and the Windows operating system, allowing agents to perform tasks such as **file navigation, application control, UI interaction, QA testing,** and more.
+
+## Updates
+
+- Try out 🪟[Windows-Use](https://github.com/CursorTouch/Windows-Use)!!, an agent built using Windows-MCP.
+- Windows-MCP is now featured as Desktop Extension in `Claude Desktop`.
+
+### Supported Operating Systems
+
+- Windows 7
+- Windows 8, 8.1
+- Windows 10
+- Windows 11  
+
+## 🎥 Demos
+
+<https://github.com/user-attachments/assets/d0e7ed1d-6189-4de6-838a-5ef8e1cad54e>
+
+<https://github.com/user-attachments/assets/d2b372dc-8d00-4d71-9677-4c64f5987485>
+
+## ✨ Key Features
+
+- **Seamless Windows Integration**  
+  Interacts natively with Windows UI elements, opens apps, controls windows, simulates user input, and more.
+
+- **Use Any LLM (Vision Optional)**
+   Unlike many automation tools, Windows MCP doesn't rely on any traditional computer vision techniques or specific fine-tuned models; it works with any LLMs, reducing complexity and setup time.
+
+- **Rich Toolset for UI Automation**  
+  Includes tools for basic keyboard, mouse operation and capturing window/UI state.
+
+- **Lightweight & Open-Source**  
+  Minimal dependencies and easy setup with full source code available under MIT license.
+
+- **Customizable & Extendable**  
+  Easily adapt or extend tools to suit your unique automation or AI integration needs.
+
+- **Real-Time Interaction**  
+  Typical latency between actions (e.g., from one mouse click to the next) ranges from **0.7 to 2.5 secs**, and may slightly vary based on the number of active applications and system load, also the inferencing speed of the llm.
+
+## 🛠️Installation
+
+### Prerequisites
+
+- Python 3.13+
+- UV (Package Manager) from Astra, install with `pip install uv` or `curl -LsSf https://astral.sh/uv/install.sh | sh`
+- `English` as the default language in Windows highly preferred or disable the `Launch-Tool` and `Switch-Tool` in the MCP Server for Windows with other languages.
+
+<details>
+  <summary>Install in Claude Desktop</summary>
+
+  1. Install [Claude Desktop](https://claude.ai/download) and
+
+```shell
+npm install -g @anthropic-ai/dxt
+```
+
+  2. Clone the repository.
+
+```shell
+git clone https://github.com/CursorTouch/Windows-MCP.git
+
+cd Windows-MCP
+```
+
+  3. Build Desktop Extension `DXT`:
+
+```shell
+npx @anthropic-ai/dxt pack
+```
+
+  4. Open Claude Desktop:
+
+Go to `Settings->Extensions->Advance Settings->Install Extension` (locate the `.dxt` file)-> Install
+
+  5. Enjoy 🥳.
+
+For additional Claude Desktop integration troubleshooting, see the [MCP documentation](https://modelcontextprotocol.io/quickstart/server#claude-for-desktop-integration-issues). The documentation includes helpful tips for checking logs and resolving common issues.
+</details>
+
+<details>
+  <summary>Install in Perplexity Desktop</summary>
+
+  1. Install [Perplexity Desktop](https://apps.microsoft.com/detail/xp8jnqfbqh6pvf):
+
+  2. Clone the repository.
+
+```shell
+git clone https://github.com/CursorTouch/Windows-MCP.git
+
+cd Windows-MCP
+```
+  
+  3. Open Perplexity Desktop:
+
+Go to `Settings->Connectors->Add Connector->Advanced`
+
+  4. Enter the name as `Windows-MCP`, then paste the following JSON in the text area.
+
+```json
+{
+  "command": "uv",
+  "args": [
+    "--directory",
+    "<path to the windows-mcp directory>",
+    "run",
+    "main.py"
+  ]
+}
+```
+
+5. Click `Save` and Enjoy 🥳.
+
+For additional Claude Desktop integration troubleshooting, see the [Perplexity MCP Support](https://www.perplexity.ai/help-center/en/articles/11502712-local-and-remote-mcps-for-perplexity). The documentation includes helpful tips for checking logs and resolving common issues.
+</details>
+
+<details>
+  <summary> Install in Gemini CLI</summary>
+
+  1. Install Gemini CLI:
+
+```shell
+npm install -g @google/gemini-cli
+```
+
+  2. Clone the repository.
+
+```shell
+git clone https://github.com/CursorTouch/Windows-MCP.git
+
+cd Windows-MCP
+```
+
+  3. Navigate to `%USERPROFILE%/.gemini` in File Explorer and open `settings.json`.
+
+  4. Add the `windows-mcp` config in the `settings.json` and save it.
+
+```json
+{
+  "theme": "Default",
+  ...
+//MCP Server Config
+  "mcpServers": {
+    "windows-mcp": {
+      "command": "uv",
+      "args": [
+        "--directory",
+        "<path to the windows-mcp directory>",
+        "run",
+        "main.py"
+      ]
+    }
+  }
+}
+```
+
+  5. Rerun Gemini CLI in terminal. Enjoy 🥳
+</details>
+
+<details>
+  <summary>Install in Qwen Code</summary>
+  1. Install Qwen Code:
+
+```shell
+npm install -g @qwen-code/qwen-code@latest
+```
+  2. Clone the repository.
+
+```shell
+git clone https://github.com/CursorTouch/Windows-MCP.git
+
+cd Windows-MCP
+```
+
+  3. Navigate to `%USERPROFILE%/.qwen/settings.json`.
+
+  4. Add the `windows-mcp` config in the `settings.json` and save it.
+
+```json
+{
+//MCP Server Config
+  "mcpServers": {
+    "windows-mcp": {
+      "command": "uv",
+      "args": [
+        "--directory",
+        "<path to the windows-mcp directory>",
+        "run",
+        "main.py"
+      ]
+    }
+  }
+}
+```
+
+  5. Rerun Qwen Code in terminal. Enjoy 🥳
+</details>
+
+<details>
+  <summary>Install in Codex CLI</summary>
+  1. Install Codex CLI:
+
+```shell
+npm install -g @openai/codex
+```
+  2. Clone the repository.
+
+```shell
+git clone https://github.com/CursorTouch/Windows-MCP.git
+
+cd Windows-MCP
+```
+  3. Navigate to `%USERPROFILE%/.codex/config.toml`.
+
+  4. Add the `windows-mcp` config in the `config.toml` and save it.
+
+```toml
+[mcp_servers.windows-mcp]
+command="uv"
+args=[
+  "--directory",
+  "<path to the windows-mcp directory>",
+  "run",
+  "main.py"
+]
+```
+
+  5. Rerun Codex CLI in terminal. Enjoy 🥳
+</details>
+
+---
+
+## 🔨MCP Tools
+
+MCP Client can access the following tools to interact with Windows:
+
+- `Click-Tool`: Click on the screen at the given coordinates.
+- `Type-Tool`: Type text on an element (optionally clears existing text).
+- `Clipboard-Tool`: Copy or paste using the system clipboard.
+- `Scroll-Tool`: Scroll vertically or horizontally on the window or specific regions.
+- `Drag-Tool`: Drag from one point to another.
+- `Move-Tool`: Move mouse pointer.
+- `Shortcut-Tool`: Press keyboard shortcuts (`Ctrl+c`, `Alt+Tab`, etc).
+- `Key-Tool`: Press a single key.
+- `Wait-Tool`: Pause for a defined duration.
+- `State-Tool`: Combined snapshot of default language, browser, active apps and interactive, textual and scrollable elements along with screenshot of the desktop.
+- `Resize-Tool`: Used to change the window size or location of an app.
+- `Launch-Tool`: To launch an application from the start menu.
+- `Shell-Tool`: To execute PowerShell commands.
+- `Scrape-Tool`: To scrape the entire webpage for information.
+
+## 🤝 Connect with Us
+Stay updated and join our community:
+
+- 📢 Follow us on [X](https://x.com/CursorTouch) for the latest news and updates
+
+- 💬 Join our [Discord Community](https://discord.com/invite/Aue9Yj2VzS)
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=CursorTouch/Windows-MCP&type=Date)](https://www.star-history.com/#CursorTouch/Windows-MCP&Date)
+
+## ⚠️Caution
+
+This MCP interacts directly with your Windows operating system to perform actions. Use with caution and avoid deploying it in environments where such risks cannot be tolerated.
+
+## 📝 Limitations
+
+- Selecting specific sections of the text in a paragraph, as the MCP is relying on a11y tree. (⌛ Working on it.)
+- `Type-Tool` is meant for typing text, not programming in IDE because of it types program as a whole in a file. (⌛ Working on it.)
+- This MCP server can't be used to play video games.
+
+## 🪪License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
+
+## 🤝Contributing
+
+Contributions are welcome! Please see [CONTRIBUTING](CONTRIBUTING) for setup instructions and development guidelines.
+
+Made with ❤️ by [CursorTouch](https://github.com/CursorTouch)
+
+## Citation
+
+```bibtex
+@software{
+  author       = {CursorTouch},
+  title        = {Windows-MCP: Lightweight open-source project for integrating LLM agents with Windows},
+  year         = {2024},
+  publisher    = {GitHub},
+  url={https://github.com/CursorTouch/Windows-MCP}
+}
+```

windows_mcp-0.3.0.dist-info/RECORD

@@ -0,0 +1,13 @@
+__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+desktop/__init__.py,sha256=7LXFkuy9gyX94MIEKEMl5kUnWfw1r7RG3mPgzy0ikTs,11121
+desktop/config.py,sha256=p7eq__X38IqQmREdm5P1pSZVWSbNI6S-xVqiRaSRjY0,293
+desktop/views.py,sha256=CsLh9K-juoEvdmmDRRt7LIY_iWVh1wdZR0WYvJ5Gy60,1002
+tree/__init__.py,sha256=ALJ6vDlRzDDIIGhzzzVqaeHdB4gomSgktYxuoTawjC8,14807
+tree/config.py,sha256=wlJvBY_7bHWgK11AGndjrp94mNOu2kXUed_b7UMwlg8,586
+tree/utils.py,sha256=6hbxdIQPrAY-I3jcHsRqodHlxboTQj2GnLA71bf1lqY,911
+tree/views.py,sha256=HPdq8rNwnxbeM6vB9v33cx-48JRWWzRWRogFQRhAcmA,2229
+windows_mcp-0.3.0.dist-info/licenses/LICENSE.md,sha256=U1UM4Xi_IX-jHnHjGT0rETNia-Ck8gd92iSQMqQ6a8Y,1089
+windows_mcp-0.3.0.dist-info/METADATA,sha256=uGiFCwztQ6P9C4Ct7gGhrO1m5WzWAiAvDHxCbK8qkvs,11674
+windows_mcp-0.3.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+windows_mcp-0.3.0.dist-info/top_level.txt,sha256=cE3tbc4LtiRWKBrK10Q4JK_yKJWkMwgOY78G1b6dhQY,22
+windows_mcp-0.3.0.dist-info/RECORD,,

windows_mcp-0.3.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.9.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

windows_mcp-0.3.0.dist-info/licenses/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 JEOMON GEORGE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

windows_mcp-0.3.0.dist-info/top_level.txt

@@ -0,0 +1,3 @@
+__init__
+desktop
+tree