More updates to itb (wip)

tools/itb/bin/itb_cursor

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+import sys
+import os
+import time
+import argparse
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.common.actions.action_builder import ActionBuilder
+from selenium.webdriver.common.actions.mouse_button import MouseButton
+
+class CursorController:
+    def __init__(self, debug_port: int, verbose: bool = False):
+        """Initialize connection to Chrome instance"""
+        self.verbose = verbose
+        options = webdriver.ChromeOptions()
+        options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
+        self.driver = webdriver.Chrome(options=options)
+        
+        # Load JavaScript files
+        js_dir = os.path.join(os.path.dirname(__file__), "..", "js")
+        self.js_code = ""
+        for filename in ["viewport.js", "mouse.js"]:
+            with open(os.path.join(js_dir, filename)) as f:
+                self.js_code += f.read() + "\n"
+
+    def log(self, msg: str):
+        """Log message if verbose mode is enabled"""
+        if self.verbose:
+            print(f"DEBUG: {msg}", file=sys.stderr)
+
+    def get_viewport_info(self):
+        """Get viewport dimensions and scroll position"""
+        return self.driver.execute_script(self.js_code + "; return getViewportInfo();")
+
+    def get_mouse_position(self):
+        """Get current mouse coordinates"""
+        return self.driver.execute_script(self.js_code + "; return getMousePosition();")
+
+    def scroll_to_target(self, x: int, y: int):
+        """Scroll viewport to make target coordinates visible"""
+        viewport = self.get_viewport_info()
+        self.driver.execute_script(
+            "window.scrollTo(arguments[0], arguments[1]);",
+            max(0, x - viewport['width'] // 2),
+            max(0, y - viewport['height'] // 2)
+        )
+        time.sleep(0.5)  # Wait for scroll to complete
+
+    def get_element_at_position(self, x: int, y: int):
+        """Get the element at specified coordinates"""
+        return self.driver.execute_script(
+            self.js_code + "; return document.elementFromPoint(arguments[0], arguments[1]);",
+            x, y
+        )
+
+    def move_cursor(self, x: int, y: int, perform_click: bool = False,
+                   double_click: bool = False, right_click: bool = False,
+                   element_id: str = None) -> bool:
+        """Move cursor to specified coordinates and optionally perform click actions"""
+        try:
+            self.log(f"Moving cursor to ({x}, {y})")
+            
+            if element_id:
+                # Try to find element by ID first
+                try:
+                    element = self.driver.find_element(By.ID, element_id)
+                    self.log(f"Found element by ID: {element_id}")
+                except:
+                    self.log(f"Failed to find element by ID: {element_id}")
+                    return False
+            else:
+                # Scroll to make target visible
+                self.scroll_to_target(x, y)
+                
+                # Get element at position
+                element = self.get_element_at_position(x, y)
+                if not element:
+                    self.log(f"No element found at coordinates ({x}, {y})")
+                    return False
+
+            # Create action chain
+            actions = ActionChains(self.driver)
+            
+            if element_id:
+                actions.move_to_element(element)
+            else:
+                actions.move_by_offset(x, y)
+
+            if perform_click:
+                actions.click()
+            elif double_click:
+                actions.double_click()
+            elif right_click:
+                actions.context_click()
+
+            # Execute the action chain
+            actions.perform()
+            return True
+
+        except Exception as e:
+            self.log(f"Error moving cursor: {str(e)}")
+            return False
+
+def main():
+    parser = argparse.ArgumentParser(description='Control mouse cursor in Chrome')
+    parser.add_argument('debug_port', type=int, help='Chrome debug port')
+    parser.add_argument('-x', type=int, help='Target X coordinate')
+    parser.add_argument('-y', type=int, help='Target Y coordinate')
+    parser.add_argument('--click', action='store_true', help='Perform click')
+    parser.add_argument('--double', action='store_true', help='Perform double click')
+    parser.add_argument('--right', action='store_true', help='Perform right click')
+    parser.add_argument('--id', help='Element ID to target')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Enable debug output')
+    
+    args = parser.parse_args()
+
+    if args.double and args.right:
+        print("Cannot specify both --double and --right", file=sys.stderr)
+        sys.exit(1)
+        
+    if not args.id and (args.x is None or args.y is None):
+        print("Must specify either element ID or both x and y coordinates", file=sys.stderr) 
+        sys.exit(1)
+        
+    try:
+        controller = CursorController(args.debug_port, args.verbose)
+        success = controller.move_cursor(
+            x=args.x if args.x is not None else 0,
+            y=args.y if args.y is not None else 0,
+            perform_click=args.click,
+            double_click=args.double,
+            right_click=args.right,
+            element_id=args.id
+        )
+        sys.exit(0 if success else 1)
+        
+    except Exception as e:
+        if args.verbose:
+            print(f"Fatal error: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

tools/itb/bin/itb_screenshot

@@ -311,4 +311,4 @@ def main():
         sys.exit(1)
 
 if __name__ == "__main__":
-    main()
+    main()

tools/itb/bin/itb_start

@@ -7,7 +7,7 @@ def start(url):
     debug_port = random.randint(9000, 9999)
     chrome_cmd = [
         '/usr/bin/google-chrome',
-        #'--headless=new',
+        '--headless=new',
         '--no-sandbox',
         '--remote-debugging-port=' + str(debug_port),
         url

tools/itb/js/dom_analyzer.js

@@ -205,4 +205,4 @@ function analyzePageContent() {
         viewport: getViewportInfo(),
         content: body ? processElements(body) : []
     };
-}
+}

tools/itb/js/dom_analyzer.js.orig

@@ -0,0 +1,208 @@
+// Helper function to check if element is clickable
+function isClickable(element) {
+    // Check tag name
+    if (element.tagName === 'BUTTON' || 
+        element.tagName === 'A' || 
+        (element.tagName === 'INPUT' && element.type === 'submit')) {
+        return true;
+    }
+    
+    // Check ARIA role
+    if (element.getAttribute('role') === 'button') {
+        return true;
+    }
+    
+    // Check for click handlers
+    if (element.onclick || 
+        element.getAttribute('onclick') || 
+        element.getAttribute('ng-click') ||
+        element.getAttribute('v-on:click')) {
+        return true;
+    }
+    
+    // Check computed style
+    const style = window.getComputedStyle(element);
+    if (style.cursor === 'pointer') {
+        return true;
+    }
+    
+    return false;
+}
+
+// Helper function to check if element is an input
+function isInputElement(element) {
+    const inputTags = ['INPUT', 'TEXTAREA', 'SELECT'];
+    if (inputTags.includes(element.tagName)) {
+        return true;
+    }
+    
+    if (element.getAttribute('contenteditable') === 'true') {
+        return true;
+    }
+    
+    if (element.getAttribute('role') === 'textbox' ||
+        element.getAttribute('role') === 'combobox') {
+        return true;
+    }
+    
+    return false;
+}
+
+// Helper function to check if element is scrollable
+function isScrollable(element) {
+    const style = window.getComputedStyle(element);
+    
+    // Check for scroll overflow
+    if (style.overflow === 'auto' || 
+        style.overflow === 'scroll' ||
+        style.overflowY === 'auto' || 
+        style.overflowY === 'scroll' ||
+        style.overflowX === 'auto' || 
+        style.overflowX === 'scroll') {
+        
+        // Verify element has scrollable content
+        return element.scrollHeight > element.clientHeight ||
+               element.scrollWidth > element.clientWidth;
+    }
+    
+    return false;
+}
+
+// Helper function to check if element is actually visible
+function isElementVisible(element) {
+    const style = window.getComputedStyle(element);
+    
+    // Special handling for fixed position elements
+    if (style.position === 'fixed') {
+        // Fixed elements are always considered visible if they have dimensions
+        if (element.offsetWidth > 0 && element.offsetHeight > 0) {
+            return true;
+        }
+    }
+    
+    // For non-fixed elements, check offsetParent
+    if (!element.offsetParent && element !== document.body) {
+        return false;
+    }
+    
+    if (style.display === 'none' || 
+        style.visibility === 'hidden' || 
+        style.opacity === '0') {
+        return false;
+    }
+    
+    const rect = element.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) {
+        return false;
+    }
+    
+    // For non-fixed elements, check viewport boundaries
+    if (style.position !== 'fixed') {
+        const viewportWidth = window.innerWidth;
+        const viewportHeight = window.innerHeight;
+        
+        if (rect.right < 0 || 
+            rect.bottom < 0 || 
+            rect.left > viewportWidth || 
+            rect.top > viewportHeight) {
+            return false;
+        }
+    }
+    
+    return true;
+}
+
+// Helper function to get element text
+function getElementText(element) {
+    // Only process direct text nodes
+    const textNodes = Array.from(element.childNodes)
+        .filter(node => node.nodeType === Node.TEXT_NODE);
+        
+    if (textNodes.length === 0) return '';
+    
+    return textNodes
+        .map(node => node.textContent.trim())
+        .filter(text => text.length > 0)
+        .join(' ');
+}
+
+function processElements(element) {
+    if (!element || !isElementVisible(element)) return [];
+    
+    const rect = element.getBoundingClientRect();
+    const styles = window.getComputedStyle(element);
+    const isFixed = styles.position === 'fixed';
+    
+    // Get element base info
+    const elementInfo = {
+        tag: element.tagName.toLowerCase(),
+        x: Math.round(isFixed ? rect.left : rect.left + window.pageXOffset),
+        y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        interactions: [],
+        fixed: isFixed
+    };
+    
+    // Add ID or suggested click coordinates for any interactive element
+    if (isClickable(element) || isInputElement(element)) {
+        const id = element.id;
+        if (id) {
+            elementInfo.id = id;
+        } else {
+            // Only add suggested click if there's no ID
+            elementInfo.suggestedClick = {
+                x: Math.round(elementInfo.x + elementInfo.width / 2),
+                y: Math.round(elementInfo.y + elementInfo.height / 2)
+            };
+        }
+    }
+    
+    // Add text only if this isn't going to be an interactive element
+    const isInteractive = isClickable(element) || isInputElement(element);
+    if (!isInteractive) {
+        elementInfo.text = getElementText(element);
+    }
+    
+    // Add interactions
+    if (isClickable(element)) {
+        elementInfo.interactions.push('click');
+        elementInfo.buttonText = getElementText(element);
+    }
+    
+    if (isInputElement(element)) {
+        const inputType = element.getAttribute('type') || 'text';
+        elementInfo.interactions.push(`input:${inputType}`);
+        
+        const placeholder = element.getAttribute('placeholder');
+        if (placeholder) {
+            elementInfo.placeholder = placeholder;
+        }
+    }
+    
+    if (isScrollable(element)) {
+        elementInfo.interactions.push('scroll');
+    }
+    
+    // Special handling for images
+    if (element.tagName === 'IMG') {
+        elementInfo.src = element.src;
+        elementInfo.alt = element.alt;
+    }
+    
+    // Process children
+    elementInfo.children = Array.from(element.children)
+        .map(child => processElements(child))
+        .flat()
+        .filter(child => child !== null);
+    
+    return elementInfo;
+}
+
+function analyzePageContent() {
+    const body = document.body;
+    return {
+        viewport: getViewportInfo(),
+        content: body ? processElements(body) : []
+    };
+}

tools/itb/js/dom_analyzer.js.patch

@@ -0,0 +1,178 @@
+136,137c136,137
+<     // Get element base info
+<     const elementInfo = {
+---
+>     // Get element base info
+>     const elementInfo = {
+140c140
+<     y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset),
+---
+>     y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset),
+141c141
+<     width: Math.round(rect.width),
+---
+>     width: Math.round(rect.width),
+142c142
+<     height: Math.round(rect.height),
+---
+>     height: Math.round(rect.height),
+143c143
+<     interactions: [],
+---
+>     interactions: [],
+144c144
+<     fixed: isFixed
+---
+>     fixed: isFixed
+148c148
+<     if (isClickable(element) || isInputElement(element)) {
+---
+>     if (isClickable(element) || isInputElement(element)) {
+150c150
+<         const id = element.id;
+---
+>         const id = element.id;
+151c151
+<         if (id) {
+---
+>         if (id) {
+152c152
+<             elementInfo.id = id;
+---
+>             elementInfo.id = id;
+153c153
+<         } else {
+---
+>         } else {
+154c154
+<             // Only add suggested click if there's no ID
+---
+>             // Only add suggested click if there's no ID
+155c155
+<             elementInfo.suggestedClick = {
+---
+>             elementInfo.suggestedClick = {
+156c156
+<                 x: Math.round(elementInfo.x + elementInfo.width / 2),
+---
+>                 x: Math.round(elementInfo.x + elementInfo.width / 2),
+157c157
+<                 y: Math.round(elementInfo.y + elementInfo.height / 2)
+---
+>                 y: Math.round(elementInfo.y + elementInfo.height / 2)
+158c158
+<             };
+---
+>             };
+161c161
+<     const isInteractive = isClickable(element) || isInputElement(element);
+---
+>     const isInteractive = isClickable(element) || isInputElement(element);
+163c163
+<     if (!isInteractive) {
+---
+>     if (!isInteractive) {
+164c164
+<         elementInfo.text = getElementText(element);
+---
+>         elementInfo.text = getElementText(element);
+166c166
+<     }
+---
+>     }
+168c168
+<     if (isClickable(element)) {
+---
+>     if (isClickable(element)) {
+169c169
+<         elementInfo.interactions.push('click');
+---
+>         elementInfo.interactions.push('click');
+170c170
+<         elementInfo.buttonText = getElementText(element);
+---
+>         elementInfo.buttonText = getElementText(element);
+172c172
+<     }
+---
+>     }
+173c173
+<     if (isInputElement(element)) {
+---
+>     if (isInputElement(element)) {
+174c174
+<         const inputType = element.getAttribute('type') || 'text';
+---
+>         const inputType = element.getAttribute('type') || 'text';
+175c175
+<         elementInfo.interactions.push(`input:${inputType}`);
+---
+>         elementInfo.interactions.push(`input:${inputType}`);
+177c177
+<         const placeholder = element.getAttribute('placeholder');
+---
+>         const placeholder = element.getAttribute('placeholder');
+178c178
+<         if (placeholder) {
+---
+>         if (placeholder) {
+179c179
+<             elementInfo.placeholder = placeholder;
+---
+>             elementInfo.placeholder = placeholder;
+180c180
+<         }
+---
+>         }
+182c182
+<     }
+---
+>     }
+183c183
+<     if (isScrollable(element)) {
+---
+>     if (isScrollable(element)) {
+184c184
+<         elementInfo.interactions.push('scroll');
+---
+>         elementInfo.interactions.push('scroll');
+186c186
+<     }
+---
+>     }
+188c188
+<     if (element.tagName === 'IMG') {
+---
+>     if (element.tagName === 'IMG') {
+189c189
+<         elementInfo.src = element.src;
+---
+>         elementInfo.src = element.src;
+190c190
+<         elementInfo.alt = element.alt;
+---
+>         elementInfo.alt = element.alt;
+192c192
+<     }
+---
+>     }
+194c194
+<     elementInfo.children = Array.from(element.children)
+---
+>     elementInfo.children = Array.from(element.children)
+195c195
+<         .map(child => processElements(child))
+---
+>         .map(child => processElements(child))
+196c196
+<         .flat()
+---
+>         .flat()
+197c197
+<         .filter(child => child !== null);
+---
+>         .filter(child => child !== null);
+199c199
+<     return elementInfo;
+---
+>     return elementInfo;

tools/itb/js/dom_analyzer.js.rej

@@ -0,0 +1,27 @@
+*** /dev/null
+--- /dev/null
+***************
+*** 140
+-     y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset),
+--- 140 -----
++     y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset),
+***************
+*** 141
+-     width: Math.round(rect.width),
+--- 141 -----
++     width: Math.round(rect.width),
+***************
+*** 142
+-     height: Math.round(rect.height),
+--- 142 -----
++     height: Math.round(rect.height),
+***************
+*** 143
+-     interactions: [],
+--- 143 -----
++     interactions: [],
+***************
+*** 144
+-     fixed: isFixed
+--- 144 -----
++     fixed: isFixed

tools/itb/setup.py

@@ -5,13 +5,14 @@ setup(
     version="0.1.0",
     packages=find_packages(),
     scripts=[
-        'bin/itb_start',
-        'bin/itb_screenshot',
-        'bin/itb_scroll',
         'bin/itb_click',
+        'bin/itb_cursor',
         'bin/itb_input',
         'bin/itb_navigate',
-        'bin/itb_refresh'
+        'bin/itb_refresh',
+        'bin/itb_screenshot',
+        'bin/itb_scroll',
+        'bin/itb_start'
     ],
     install_requires=[
         'selenium>=4.0.0',