Merge branch 'master' of https://git.nielsgeens.be/llm/SIA

tools/itb/README.md

@@ -64,13 +64,12 @@ itb_scroll <session_id> <x> <y>
 
 ## itb_click
 
-Clicks on a specified element in the browser.
+Clicks on a specified viewport coordinates
 
 ### Usage
 
 ```
-itb_click <session_id> -e <element_id>
-itb_click <session_id> -x <x> -y <y>
+itb_click <session_id> -x <x> -y <y> [--right] [--double]
 ```
 
 ### Implementation Strategies

tools/itb/bin/itb_click

@@ -0,0 +1,159 @@
+#!/usr/bin/env python3
+import sys
+import time
+import random
+import argparse
+import json
+from selenium import webdriver
+from selenium.webdriver import ActionChains
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from selenium.webdriver.common.actions.mouse_button import MouseButton
+from selenium.webdriver.common.by import By
+
+def debug(msg: str):
+    """Print debug message to stderr"""
+    print(f"DEBUG: {msg}", file=sys.stderr)
+
+class ClickExecutor:
+    def __init__(self, debug_port: int):
+        debug(f"Connecting to Chrome on debug port {debug_port}")
+        options = webdriver.ChromeOptions()
+        options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
+        self.driver = webdriver.Chrome(options=options)
+        
+        # Get initial window info
+        self.window_size = self.driver.get_window_size()
+        debug(f"Window size: {json.dumps(self.window_size)}")
+        
+        self.viewport_size = self.get_viewport_size()
+        debug(f"Viewport size: {json.dumps(self.viewport_size)}")
+        
+        self.scroll_position = self.get_scroll_position()
+        debug(f"Scroll position: {json.dumps(self.scroll_position)}")
+
+    def get_viewport_size(self):
+        """Get viewport dimensions"""
+        return self.driver.execute_script("""
+            return {
+                width: window.innerWidth,
+                height: window.innerHeight,
+                pageYOffset: window.pageYOffset,
+                pageXOffset: window.pageXOffset
+            };
+        """)
+        
+    def get_scroll_position(self):
+        """Get current scroll position"""
+        return self.driver.execute_script("""
+            return {
+                scrollX: window.scrollX,
+                scrollY: window.scrollY,
+                pageXOffset: window.pageXOffset,
+                pageYOffset: window.pageYOffset
+            };
+        """)
+        
+    def get_mouse_position(self):
+        """Get current mouse coordinates"""
+        return self.driver.execute_script("""
+            return {
+                screenX: window.screenX,
+                screenY: window.screenY,
+                clientX: window.event ? window.event.clientX : 0,
+                clientY: window.event ? window.event.clientY : 0
+            };
+        """)
+
+    def scroll_to_target(self, x: int, y: int):
+        """Scroll to make target coordinates visible"""
+        self.driver.execute_script(
+            "window.scrollTo(arguments[0], arguments[1]);", 
+            max(0, x - self.viewport_size['width'] // 2),
+            max(0, y - self.viewport_size['height'] // 2)
+        )
+        # Wait for scroll to complete
+        time.sleep(0.5)
+        self.scroll_position = self.get_scroll_position()
+        debug(f"New scroll position: {json.dumps(self.scroll_position)}")
+
+    def get_element_at_position(self, x: int, y: int):
+        """Get the element at the specified coordinates"""
+        return self.driver.execute_script("""
+            return document.elementFromPoint(arguments[0], arguments[1]);
+        """, x, y)
+
+    def execute_click(self, x: int, y: int, double: bool = False, right: bool = False):
+        """Execute click operation with improved coordinate handling"""
+        try:
+            debug(f"Attempting to click at coordinates ({x}, {y})")
+            debug(f"Click type: {'double' if double else 'right' if right else 'single'}")
+            
+            # First scroll to make target visible
+            self.scroll_to_target(x, y)
+            
+            # Get the element at the target position
+            element = self.get_element_at_position(x, y)
+            if not element:
+                debug(f"No element found at coordinates ({x}, {y})")
+                return False
+                
+            # Try clicking the element directly
+            try:
+                debug("Attempting to click element directly")
+                element = self.driver.execute_script("return arguments[0];", element)
+                
+                actions = ActionChains(self.driver)
+                actions.move_to_element(element)
+                
+                if right:
+                    actions.context_click()
+                elif double:
+                    actions.double_click()
+                else:
+                    actions.click()
+                    
+                actions.perform()
+                return True
+                
+            except Exception as e:
+                debug(f"Direct element click failed: {str(e)}")
+                debug("Falling back to JavaScript click")
+                
+                # Fallback to JavaScript click
+                try:
+                    self.driver.execute_script("arguments[0].click();", element)
+                    return True
+                except Exception as js_error:
+                    debug(f"JavaScript click failed: {str(js_error)}")
+                    return False
+            
+        except Exception as e:
+            debug(f"Click operation failed with error: {str(e)}")
+            return False
+
+def main():
+    parser = argparse.ArgumentParser(description='Execute mouse clicks in Chrome')
+    parser.add_argument('debug_port', type=int, help='Chrome debug port')
+    parser.add_argument('-x', type=int, required=True, help='X coordinate')
+    parser.add_argument('-y', type=int, required=True, help='Y coordinate')
+    parser.add_argument('--double', action='store_true', help='Perform double click')
+    parser.add_argument('--right', action='store_true', help='Perform right click')
+    
+    args = parser.parse_args()
+    
+    if args.double and args.right:
+        print("Cannot specify both --double and --right", file=sys.stderr)
+        sys.exit(1)
+        
+    try:
+        debug("Initializing ClickExecutor")
+        executor = ClickExecutor(args.debug_port)
+        success = executor.execute_click(args.x, args.y, args.double, args.right)
+        sys.exit(0 if success else 1)
+        
+    except Exception as e:
+        debug(f"Fatal error: {str(e)}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()