diff --git a/tools/itb/bin/itb_cursor b/tools/itb/bin/itb_cursor new file mode 100644 index 0000000..d4afb85 --- /dev/null +++ b/tools/itb/bin/itb_cursor @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +import sys +import os +import time +import argparse +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.common.actions.action_builder import ActionBuilder +from selenium.webdriver.common.actions.mouse_button import MouseButton + +class CursorController: + def __init__(self, debug_port: int, verbose: bool = False): + """Initialize connection to Chrome instance""" + self.verbose = verbose + options = webdriver.ChromeOptions() + options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}") + self.driver = webdriver.Chrome(options=options) + + # Load JavaScript files + js_dir = os.path.join(os.path.dirname(__file__), "..", "js") + self.js_code = "" + for filename in ["viewport.js", "mouse.js"]: + with open(os.path.join(js_dir, filename)) as f: + self.js_code += f.read() + "\n" + + def log(self, msg: str): + """Log message if verbose mode is enabled""" + if self.verbose: + print(f"DEBUG: {msg}", file=sys.stderr) + + def get_viewport_info(self): + """Get viewport dimensions and scroll position""" + return self.driver.execute_script(self.js_code + "; return getViewportInfo();") + + def get_mouse_position(self): + """Get current mouse coordinates""" + return self.driver.execute_script(self.js_code + "; return getMousePosition();") + + def scroll_to_target(self, x: int, y: int): + """Scroll viewport to make target coordinates visible""" + viewport = self.get_viewport_info() + self.driver.execute_script( + "window.scrollTo(arguments[0], arguments[1]);", + max(0, x - viewport['width'] // 2), + max(0, y - viewport['height'] // 2) + ) + time.sleep(0.5) # Wait for scroll to complete + + def get_element_at_position(self, x: int, y: int): + """Get the element at specified coordinates""" + return self.driver.execute_script( + self.js_code + "; return document.elementFromPoint(arguments[0], arguments[1]);", + x, y + ) + + def move_cursor(self, x: int, y: int, perform_click: bool = False, + double_click: bool = False, right_click: bool = False, + element_id: str = None) -> bool: + """Move cursor to specified coordinates and optionally perform click actions""" + try: + self.log(f"Moving cursor to ({x}, {y})") + + if element_id: + # Try to find element by ID first + try: + element = self.driver.find_element(By.ID, element_id) + self.log(f"Found element by ID: {element_id}") + except: + self.log(f"Failed to find element by ID: {element_id}") + return False + else: + # Scroll to make target visible + self.scroll_to_target(x, y) + + # Get element at position + element = self.get_element_at_position(x, y) + if not element: + self.log(f"No element found at coordinates ({x}, {y})") + return False + + # Create action chain + actions = ActionChains(self.driver) + + if element_id: + actions.move_to_element(element) + else: + actions.move_by_offset(x, y) + + if perform_click: + actions.click() + elif double_click: + actions.double_click() + elif right_click: + actions.context_click() + + # Execute the action chain + actions.perform() + return True + + except Exception as e: + self.log(f"Error moving cursor: {str(e)}") + return False + +def main(): + parser = argparse.ArgumentParser(description='Control mouse cursor in Chrome') + parser.add_argument('debug_port', type=int, help='Chrome debug port') + parser.add_argument('-x', type=int, help='Target X coordinate') + parser.add_argument('-y', type=int, help='Target Y coordinate') + parser.add_argument('--click', action='store_true', help='Perform click') + parser.add_argument('--double', action='store_true', help='Perform double click') + parser.add_argument('--right', action='store_true', help='Perform right click') + parser.add_argument('--id', help='Element ID to target') + parser.add_argument('-v', '--verbose', action='store_true', help='Enable debug output') + + args = parser.parse_args() + + if args.double and args.right: + print("Cannot specify both --double and --right", file=sys.stderr) + sys.exit(1) + + if not args.id and (args.x is None or args.y is None): + print("Must specify either element ID or both x and y coordinates", file=sys.stderr) + sys.exit(1) + + try: + controller = CursorController(args.debug_port, args.verbose) + success = controller.move_cursor( + x=args.x if args.x is not None else 0, + y=args.y if args.y is not None else 0, + perform_click=args.click, + double_click=args.double, + right_click=args.right, + element_id=args.id + ) + sys.exit(0 if success else 1) + + except Exception as e: + if args.verbose: + print(f"Fatal error: {str(e)}", file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/itb/bin/itb_screenshot b/tools/itb/bin/itb_screenshot index 9df14c3..55e26f8 100755 --- a/tools/itb/bin/itb_screenshot +++ b/tools/itb/bin/itb_screenshot @@ -311,4 +311,4 @@ def main(): sys.exit(1) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/tools/itb/bin/itb_start b/tools/itb/bin/itb_start index 996936d..405560b 100755 --- a/tools/itb/bin/itb_start +++ b/tools/itb/bin/itb_start @@ -7,7 +7,7 @@ def start(url): debug_port = random.randint(9000, 9999) chrome_cmd = [ '/usr/bin/google-chrome', - #'--headless=new', + '--headless=new', '--no-sandbox', '--remote-debugging-port=' + str(debug_port), url diff --git a/tools/itb/js/dom_analyzer.js b/tools/itb/js/dom_analyzer.js index 72efe73..0cf9cc7 100644 --- a/tools/itb/js/dom_analyzer.js +++ b/tools/itb/js/dom_analyzer.js @@ -205,4 +205,4 @@ function analyzePageContent() { viewport: getViewportInfo(), content: body ? processElements(body) : [] }; -} \ No newline at end of file +} diff --git a/tools/itb/js/dom_analyzer.js.orig b/tools/itb/js/dom_analyzer.js.orig new file mode 100644 index 0000000..72efe73 --- /dev/null +++ b/tools/itb/js/dom_analyzer.js.orig @@ -0,0 +1,208 @@ +// Helper function to check if element is clickable +function isClickable(element) { + // Check tag name + if (element.tagName === 'BUTTON' || + element.tagName === 'A' || + (element.tagName === 'INPUT' && element.type === 'submit')) { + return true; + } + + // Check ARIA role + if (element.getAttribute('role') === 'button') { + return true; + } + + // Check for click handlers + if (element.onclick || + element.getAttribute('onclick') || + element.getAttribute('ng-click') || + element.getAttribute('v-on:click')) { + return true; + } + + // Check computed style + const style = window.getComputedStyle(element); + if (style.cursor === 'pointer') { + return true; + } + + return false; +} + +// Helper function to check if element is an input +function isInputElement(element) { + const inputTags = ['INPUT', 'TEXTAREA', 'SELECT']; + if (inputTags.includes(element.tagName)) { + return true; + } + + if (element.getAttribute('contenteditable') === 'true') { + return true; + } + + if (element.getAttribute('role') === 'textbox' || + element.getAttribute('role') === 'combobox') { + return true; + } + + return false; +} + +// Helper function to check if element is scrollable +function isScrollable(element) { + const style = window.getComputedStyle(element); + + // Check for scroll overflow + if (style.overflow === 'auto' || + style.overflow === 'scroll' || + style.overflowY === 'auto' || + style.overflowY === 'scroll' || + style.overflowX === 'auto' || + style.overflowX === 'scroll') { + + // Verify element has scrollable content + return element.scrollHeight > element.clientHeight || + element.scrollWidth > element.clientWidth; + } + + return false; +} + +// Helper function to check if element is actually visible +function isElementVisible(element) { + const style = window.getComputedStyle(element); + + // Special handling for fixed position elements + if (style.position === 'fixed') { + // Fixed elements are always considered visible if they have dimensions + if (element.offsetWidth > 0 && element.offsetHeight > 0) { + return true; + } + } + + // For non-fixed elements, check offsetParent + if (!element.offsetParent && element !== document.body) { + return false; + } + + if (style.display === 'none' || + style.visibility === 'hidden' || + style.opacity === '0') { + return false; + } + + const rect = element.getBoundingClientRect(); + if (rect.width === 0 || rect.height === 0) { + return false; + } + + // For non-fixed elements, check viewport boundaries + if (style.position !== 'fixed') { + const viewportWidth = window.innerWidth; + const viewportHeight = window.innerHeight; + + if (rect.right < 0 || + rect.bottom < 0 || + rect.left > viewportWidth || + rect.top > viewportHeight) { + return false; + } + } + + return true; +} + +// Helper function to get element text +function getElementText(element) { + // Only process direct text nodes + const textNodes = Array.from(element.childNodes) + .filter(node => node.nodeType === Node.TEXT_NODE); + + if (textNodes.length === 0) return ''; + + return textNodes + .map(node => node.textContent.trim()) + .filter(text => text.length > 0) + .join(' '); +} + +function processElements(element) { + if (!element || !isElementVisible(element)) return []; + + const rect = element.getBoundingClientRect(); + const styles = window.getComputedStyle(element); + const isFixed = styles.position === 'fixed'; + + // Get element base info + const elementInfo = { + tag: element.tagName.toLowerCase(), + x: Math.round(isFixed ? rect.left : rect.left + window.pageXOffset), + y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset), + width: Math.round(rect.width), + height: Math.round(rect.height), + interactions: [], + fixed: isFixed + }; + + // Add ID or suggested click coordinates for any interactive element + if (isClickable(element) || isInputElement(element)) { + const id = element.id; + if (id) { + elementInfo.id = id; + } else { + // Only add suggested click if there's no ID + elementInfo.suggestedClick = { + x: Math.round(elementInfo.x + elementInfo.width / 2), + y: Math.round(elementInfo.y + elementInfo.height / 2) + }; + } + } + + // Add text only if this isn't going to be an interactive element + const isInteractive = isClickable(element) || isInputElement(element); + if (!isInteractive) { + elementInfo.text = getElementText(element); + } + + // Add interactions + if (isClickable(element)) { + elementInfo.interactions.push('click'); + elementInfo.buttonText = getElementText(element); + } + + if (isInputElement(element)) { + const inputType = element.getAttribute('type') || 'text'; + elementInfo.interactions.push(`input:${inputType}`); + + const placeholder = element.getAttribute('placeholder'); + if (placeholder) { + elementInfo.placeholder = placeholder; + } + } + + if (isScrollable(element)) { + elementInfo.interactions.push('scroll'); + } + + // Special handling for images + if (element.tagName === 'IMG') { + elementInfo.src = element.src; + elementInfo.alt = element.alt; + } + + // Process children + elementInfo.children = Array.from(element.children) + .map(child => processElements(child)) + .flat() + .filter(child => child !== null); + + return elementInfo; +} + +function analyzePageContent() { + const body = document.body; + return { + viewport: getViewportInfo(), + content: body ? processElements(body) : [] + }; +} \ No newline at end of file diff --git a/tools/itb/js/dom_analyzer.js.patch b/tools/itb/js/dom_analyzer.js.patch new file mode 100644 index 0000000..80146e7 --- /dev/null +++ b/tools/itb/js/dom_analyzer.js.patch @@ -0,0 +1,178 @@ +136,137c136,137 +< // Get element base info +< const elementInfo = { +--- +> // Get element base info +> const elementInfo = { +140c140 +< y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset), +--- +> y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset), +141c141 +< width: Math.round(rect.width), +--- +> width: Math.round(rect.width), +142c142 +< height: Math.round(rect.height), +--- +> height: Math.round(rect.height), +143c143 +< interactions: [], +--- +> interactions: [], +144c144 +< fixed: isFixed +--- +> fixed: isFixed +148c148 +< if (isClickable(element) || isInputElement(element)) { +--- +> if (isClickable(element) || isInputElement(element)) { +150c150 +< const id = element.id; +--- +> const id = element.id; +151c151 +< if (id) { +--- +> if (id) { +152c152 +< elementInfo.id = id; +--- +> elementInfo.id = id; +153c153 +< } else { +--- +> } else { +154c154 +< // Only add suggested click if there's no ID +--- +> // Only add suggested click if there's no ID +155c155 +< elementInfo.suggestedClick = { +--- +> elementInfo.suggestedClick = { +156c156 +< x: Math.round(elementInfo.x + elementInfo.width / 2), +--- +> x: Math.round(elementInfo.x + elementInfo.width / 2), +157c157 +< y: Math.round(elementInfo.y + elementInfo.height / 2) +--- +> y: Math.round(elementInfo.y + elementInfo.height / 2) +158c158 +< }; +--- +> }; +161c161 +< const isInteractive = isClickable(element) || isInputElement(element); +--- +> const isInteractive = isClickable(element) || isInputElement(element); +163c163 +< if (!isInteractive) { +--- +> if (!isInteractive) { +164c164 +< elementInfo.text = getElementText(element); +--- +> elementInfo.text = getElementText(element); +166c166 +< } +--- +> } +168c168 +< if (isClickable(element)) { +--- +> if (isClickable(element)) { +169c169 +< elementInfo.interactions.push('click'); +--- +> elementInfo.interactions.push('click'); +170c170 +< elementInfo.buttonText = getElementText(element); +--- +> elementInfo.buttonText = getElementText(element); +172c172 +< } +--- +> } +173c173 +< if (isInputElement(element)) { +--- +> if (isInputElement(element)) { +174c174 +< const inputType = element.getAttribute('type') || 'text'; +--- +> const inputType = element.getAttribute('type') || 'text'; +175c175 +< elementInfo.interactions.push(`input:${inputType}`); +--- +> elementInfo.interactions.push(`input:${inputType}`); +177c177 +< const placeholder = element.getAttribute('placeholder'); +--- +> const placeholder = element.getAttribute('placeholder'); +178c178 +< if (placeholder) { +--- +> if (placeholder) { +179c179 +< elementInfo.placeholder = placeholder; +--- +> elementInfo.placeholder = placeholder; +180c180 +< } +--- +> } +182c182 +< } +--- +> } +183c183 +< if (isScrollable(element)) { +--- +> if (isScrollable(element)) { +184c184 +< elementInfo.interactions.push('scroll'); +--- +> elementInfo.interactions.push('scroll'); +186c186 +< } +--- +> } +188c188 +< if (element.tagName === 'IMG') { +--- +> if (element.tagName === 'IMG') { +189c189 +< elementInfo.src = element.src; +--- +> elementInfo.src = element.src; +190c190 +< elementInfo.alt = element.alt; +--- +> elementInfo.alt = element.alt; +192c192 +< } +--- +> } +194c194 +< elementInfo.children = Array.from(element.children) +--- +> elementInfo.children = Array.from(element.children) +195c195 +< .map(child => processElements(child)) +--- +> .map(child => processElements(child)) +196c196 +< .flat() +--- +> .flat() +197c197 +< .filter(child => child !== null); +--- +> .filter(child => child !== null); +199c199 +< return elementInfo; +--- +> return elementInfo; diff --git a/tools/itb/js/dom_analyzer.js.rej b/tools/itb/js/dom_analyzer.js.rej new file mode 100644 index 0000000..9efae9a --- /dev/null +++ b/tools/itb/js/dom_analyzer.js.rej @@ -0,0 +1,27 @@ +*** /dev/null +--- /dev/null +*************** +*** 140 +- y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset), +--- 140 ----- ++ y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset), +*************** +*** 141 +- width: Math.round(rect.width), +--- 141 ----- ++ width: Math.round(rect.width), +*************** +*** 142 +- height: Math.round(rect.height), +--- 142 ----- ++ height: Math.round(rect.height), +*************** +*** 143 +- interactions: [], +--- 143 ----- ++ interactions: [], +*************** +*** 144 +- fixed: isFixed +--- 144 ----- ++ fixed: isFixed diff --git a/tools/itb/setup.py b/tools/itb/setup.py index 6963bf9..d689c9e 100644 --- a/tools/itb/setup.py +++ b/tools/itb/setup.py @@ -5,13 +5,14 @@ setup( version="0.1.0", packages=find_packages(), scripts=[ - 'bin/itb_start', - 'bin/itb_screenshot', - 'bin/itb_scroll', 'bin/itb_click', + 'bin/itb_cursor', 'bin/itb_input', 'bin/itb_navigate', - 'bin/itb_refresh' + 'bin/itb_refresh', + 'bin/itb_screenshot', + 'bin/itb_scroll', + 'bin/itb_start' ], install_requires=[ 'selenium>=4.0.0',