Update to itb_screenshot and friends

tools/itb/bin/itb_click

@@ -2,22 +2,17 @@
 import sys
 import os
 import time
-import random
 import argparse
-import json
 from selenium import webdriver
 from selenium.webdriver import ActionChains
 from selenium.webdriver.common.actions.action_builder import ActionBuilder
 from selenium.webdriver.common.actions.mouse_button import MouseButton
 from selenium.webdriver.common.by import By
 
-def debug(msg: str):
-    """Print debug message to stderr"""
-    print(f"DEBUG: {msg}", file=sys.stderr)
-
 class ClickExecutor:
-    def __init__(self, debug_port: int):
-        debug(f"Connecting to Chrome on debug port {debug_port}")
+    def __init__(self, debug_port: int, verbose: bool = False):
+        """Initialize connection to Chrome instance"""
+        self.verbose = verbose
         options = webdriver.ChromeOptions()
         options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
         self.driver = webdriver.Chrome(options=options)
@@ -29,15 +24,15 @@ class ClickExecutor:
             with open(os.path.join(js_dir, filename)) as f:
                 self.js_code += f.read() + "\n"
         
-        # Get initial window info
+        # Get window and viewport info
         self.window_size = self.driver.get_window_size()
-        debug(f"Window size: {json.dumps(self.window_size)}")
-        
         self.viewport_size = self.get_viewport_size()
-        debug(f"Viewport size: {json.dumps(self.viewport_size)}")
-        
         self.scroll_position = self.get_scroll_position()
-        debug(f"Scroll position: {json.dumps(self.scroll_position)}")
+
+    def log(self, msg: str):
+        """Log message if verbose mode is enabled"""
+        if self.verbose:
+            print(f"DEBUG: {msg}", file=sys.stderr)
 
     def get_viewport_size(self):
         """Get viewport dimensions"""
@@ -58,37 +53,47 @@ class ClickExecutor:
             max(0, x - self.viewport_size['width'] // 2),
             max(0, y - self.viewport_size['height'] // 2)
         )
-        # Wait for scroll to complete
         time.sleep(0.5)
         self.scroll_position = self.get_scroll_position()
-        debug(f"New scroll position: {json.dumps(self.scroll_position)}")
 
     def get_element_at_position(self, x: int, y: int):
         """Get the element at the specified coordinates"""
-        return self.driver.execute_script(self.js_code + "; return getElementAtPosition(arguments[0], arguments[1]);", x, y)
+        return self.driver.execute_script(
+            self.js_code + "; return getElementAtPosition(arguments[0], arguments[1]);", 
+            x, y
+        )
 
-    def execute_click(self, x: int, y: int, double: bool = False, right: bool = False):
-        """Execute click operation with improved coordinate handling"""
+    def execute_click(self, x: int, y: int, double: bool = False, right: bool = False, element_id: str = None):
+        """Execute click operation with improved handling"""
         try:
-            debug(f"Attempting to click at coordinates ({x}, {y})")
-            debug(f"Click type: {'double' if double else 'right' if right else 'single'}")
+            self.log(f"Attempting {'double' if double else 'right' if right else 'single'} click at ({x}, {y})")
             
-            # First scroll to make target visible
-            self.scroll_to_target(x, y)
-            
-            # Get the element at the target position
-            element = self.get_element_at_position(x, y)
-            if not element:
-                debug(f"No element found at coordinates ({x}, {y})")
-                return False
+            if element_id:
+                # Try to find element by ID first
+                try:
+                    element = self.driver.find_element(By.ID, element_id)
+                    self.log(f"Found element by ID: {element_id}")
+                except:
+                    self.log(f"Failed to find element by ID: {element_id}")
+                    return False
+            else:
+                # Scroll to make target visible
+                self.scroll_to_target(x, y)
                 
-            # Try clicking the element directly
+                # Get element at position
+                element = self.get_element_at_position(x, y)
+                if not element:
+                    self.log(f"No element found at coordinates ({x}, {y})")
+                    return False
+            
+            # Try clicking the element
             try:
-                debug("Attempting to click element directly")
-                element = self.driver.execute_script("return arguments[0];", element)
-                
                 actions = ActionChains(self.driver)
-                actions.move_to_element(element)
+                
+                if element_id:
+                    actions.move_to_element(element)
+                else:
+                    actions.move_by_offset(x, y)
                 
                 if right:
                     actions.context_click()
@@ -101,28 +106,29 @@ class ClickExecutor:
                 return True
                 
             except Exception as e:
-                debug(f"Direct element click failed: {str(e)}")
-                debug("Falling back to JavaScript click")
+                self.log(f"Direct click failed: {str(e)}")
+                self.log("Attempting JavaScript click fallback")
                 
-                # Fallback to JavaScript click
                 try:
                     self.driver.execute_script("arguments[0].click();", element)
                     return True
                 except Exception as js_error:
-                    debug(f"JavaScript click failed: {str(js_error)}")
+                    self.log(f"JavaScript click failed: {str(js_error)}")
                     return False
             
         except Exception as e:
-            debug(f"Click operation failed with error: {str(e)}")
+            self.log(f"Click operation failed: {str(e)}")
             return False
 
 def main():
     parser = argparse.ArgumentParser(description='Execute mouse clicks in Chrome')
     parser.add_argument('debug_port', type=int, help='Chrome debug port')
-    parser.add_argument('-x', type=int, required=True, help='X coordinate')
-    parser.add_argument('-y', type=int, required=True, help='Y coordinate')
+    parser.add_argument('-x', type=int, help='X coordinate')
+    parser.add_argument('-y', type=int, help='Y coordinate')
     parser.add_argument('--double', action='store_true', help='Perform double click')
     parser.add_argument('--right', action='store_true', help='Perform right click')
+    parser.add_argument('--id', help='Element ID to click')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Enable debug output')
     
     args = parser.parse_args()
     
@@ -130,15 +136,25 @@ def main():
         print("Cannot specify both --double and --right", file=sys.stderr)
         sys.exit(1)
         
+    if not args.id and (args.x is None or args.y is None):
+        print("Must specify either element ID or both x and y coordinates", file=sys.stderr)
+        sys.exit(1)
+        
     try:
-        debug("Initializing ClickExecutor")
-        executor = ClickExecutor(args.debug_port)
-        success = executor.execute_click(args.x, args.y, args.double, args.right)
+        executor = ClickExecutor(args.debug_port, args.verbose)
+        success = executor.execute_click(
+            x=args.x if args.x is not None else 0,
+            y=args.y if args.y is not None else 0,
+            double=args.double,
+            right=args.right,
+            element_id=args.id
+        )
         sys.exit(0 if success else 1)
         
     except Exception as e:
-        debug(f"Fatal error: {str(e)}")
+        if args.verbose:
+            print(f"Fatal error: {str(e)}", file=sys.stderr)
         sys.exit(1)
 
 if __name__ == "__main__":
-    main()
+    main()

tools/itb/bin/itb_input

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+import sys
+import os
+import time
+import argparse
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.common.action_chains import ActionChains
+
+class InputController:
+    def __init__(self, debug_port: int, verbose: bool = False):
+        """Initialize connection to Chrome instance"""
+        self.verbose = verbose
+        options = webdriver.ChromeOptions()
+        options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
+        self.driver = webdriver.Chrome(options=options)
+
+    def log(self, msg: str):
+        """Log message if verbose mode is enabled"""
+        if self.verbose:
+            print(f"DEBUG: {msg}", file=sys.stderr)
+
+    def focus_element(self, element_id: str = None):
+        """Focus on an input element"""
+        try:
+            if element_id:
+                element = self.driver.find_element(By.ID, element_id)
+            else:
+                # Get currently focused element
+                element = self.driver.switch_to.active_element
+                
+            element.click()
+            return element
+            
+        except Exception as e:
+            self.log(f"Failed to focus element: {str(e)}")
+            return None
+
+    def enter_text(self, text: str, element_id: str = None, append: bool = False):
+        """Enter text into an input element"""
+        try:
+            element = self.focus_element(element_id)
+            if not element:
+                return False
+                
+            if not append:
+                # Clear existing text first
+                if element.tag_name.lower() in ['input', 'textarea']:
+                    element.clear()
+                else:
+                    # For contenteditable elements
+                    element.send_keys(Keys.CONTROL + 'a')
+                    element.send_keys(Keys.DELETE)
+            
+            # Type the text
+            element.send_keys(text)
+            return True
+            
+        except Exception as e:
+            self.log(f"Failed to enter text: {str(e)}")
+            return False
+
+    def set_caret_position(self, position: int, element_id: str = None):
+        """Set the text cursor position"""
+        try:
+            element = self.focus_element(element_id)
+            if not element:
+                return False
+                
+            # Use JavaScript to set caret position
+            self.driver.execute_script(
+                """
+                const el = arguments[0];
+                const pos = arguments[1];
+                if (el.setSelectionRange) {
+                    el.setSelectionRange(pos, pos);
+                } else if (el.createTextRange) {
+                    const range = el.createTextRange();
+                    range.collapse(true);
+                    range.moveEnd('character', pos);
+                    range.moveStart('character', pos);
+                    range.select();
+                }
+                """,
+                element, position
+            )
+            return True
+            
+        except Exception as e:
+            self.log(f"Failed to set caret position: {str(e)}")
+            return False
+
+    def select_text(self, start: int, end: int, element_id: str = None):
+        """Select text range in an input element"""
+        try:
+            element = self.focus_element(element_id)
+            if not element:
+                return False
+                
+            # Use JavaScript to set selection
+            self.driver.execute_script(
+                """
+                const el = arguments[0];
+                const start = arguments[1];
+                const end = arguments[2];
+                if (el.setSelectionRange) {
+                    el.setSelectionRange(start, end);
+                } else if (el.createTextRange) {
+                    const range = el.createTextRange();
+                    range.collapse(true);
+                    range.moveStart('character', start);
+                    range.moveEnd('character', end);
+                    range.select();
+                }
+                """,
+                element, start, end
+            )
+            return True
+            
+        except Exception as e:
+            self.log(f"Failed to select text: {str(e)}")
+            return False
+
+def main():
+    parser = argparse.ArgumentParser(description='Control text input in Chrome')
+    parser.add_argument('debug_port', type=int, help='Chrome debug port')
+    parser.add_argument('-i', '--id', help='Element ID')
+    parser.add_argument('-t', '--text', help='Text to input')
+    parser.add_argument('--append', action='store_true', help='Append text instead of replacing')
+    parser.add_argument('--caret', type=int, help='Set caret position')
+    parser.add_argument('--select-start', type=int, help='Text selection start')
+    parser.add_argument('--select-end', type=int, help='Text selection end')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Enable debug output')
+    
+    args = parser.parse_args()
+    
+    # Validate args
+    if args.select_start is not None and args.select_end is None:
+        print("Must specify both --select-start and --select-end", file=sys.stderr)
+        sys.exit(1)
+        
+    if args.select_end is not None and args.select_start is None:
+        print("Must specify both --select-start and --select-end", file=sys.stderr)
+        sys.exit(1)
+        
+    try:
+        controller = InputController(args.debug_port, args.verbose)
+        success = False
+        
+        if args.text is not None:
+            success = controller.enter_text(args.text, args.id, args.append)
+        elif args.caret is not None:
+            success = controller.set_caret_position(args.caret, args.id)
+        elif args.select_start is not None:
+            success = controller.select_text(args.select_start, args.select_end, args.id)
+        else:
+            success = controller.focus_element(args.id) is not None
+            
+        sys.exit(0 if success else 1)
+        
+    except Exception as e:
+        if args.verbose:
+            print(f"Fatal error: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

tools/itb/bin/itb_navigate

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+import sys
+import os
+import time
+import argparse
+from selenium import webdriver
+
+class NavigationController:
+    def __init__(self, debug_port: int):
+        """Initialize connection to Chrome instance"""
+        options = webdriver.ChromeOptions()
+        options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
+        self.driver = webdriver.Chrome(options=options)
+        
+        # Load JavaScript files
+        js_dir = os.path.join(os.path.dirname(__file__), "..", "js")
+        self.js_code = ""
+        with open(os.path.join(js_dir, "viewport.js")) as f:
+            self.js_code = f.read()
+            
+    def navigate(self, url: str) -> bool:
+        """
+        Navigate to the specified URL
+        Returns True if navigation was successful
+        """
+        try:
+            # Store initial URL for comparison
+            initial_url = self.driver.current_url
+            
+            # Execute navigation
+            self.driver.get(url)
+            
+            # Wait for navigation to complete
+            max_wait = 10  # seconds
+            start_time = time.time()
+            
+            while time.time() - start_time < max_wait:
+                # Check if URL has changed
+                if self.driver.current_url != initial_url:
+                    # Wait a bit more for page to stabilize
+                    time.sleep(0.5)
+                    return True
+                    
+                time.sleep(0.1)
+                
+            # If we get here, navigation may have failed
+            print(f"Warning: Navigation timeout after {max_wait} seconds", file=sys.stderr)
+            return False
+            
+        except Exception as e:
+            print(f"Error during navigation: {str(e)}", file=sys.stderr)
+            return False
+
+def main():
+    parser = argparse.ArgumentParser(description='Navigate to URL in Chrome')
+    parser.add_argument('debug_port', type=int, help='Chrome debug port')
+    parser.add_argument('url', help='URL to navigate to')
+    
+    args = parser.parse_args()
+    
+    try:
+        controller = NavigationController(args.debug_port)
+        success = controller.navigate(args.url)
+        sys.exit(0 if success else 1)
+        
+    except Exception as e:
+        print(f"Fatal error: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

tools/itb/bin/itb_refresh

@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+import sys
+import os
+import time
+import argparse
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+
+class RefreshController:
+    def __init__(self, debug_port: int, verbose: bool = False):
+        """Initialize connection to Chrome instance"""
+        self.verbose = verbose
+        options = webdriver.ChromeOptions()
+        options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
+        self.driver = webdriver.Chrome(options=options)
+        
+        # Load JavaScript files
+        js_dir = os.path.join(os.path.dirname(__file__), "..", "js")
+        self.js_code = ""
+        with open(os.path.join(js_dir, "viewport.js")) as f:
+            self.js_code = f.read()
+
+    def log(self, msg: str):
+        """Log message if verbose mode is enabled"""
+        if self.verbose:
+            print(f"DEBUG: {msg}", file=sys.stderr)
+
+    def get_page_state(self) -> dict:
+        """Get current page state for comparison"""
+        return {
+            'url': self.driver.current_url,
+            'title': self.driver.title,
+            'scroll_position': self.driver.execute_script(
+                self.js_code + "; return getViewportInfo();"
+            )
+        }
+
+    def wait_for_page_change(self, original_state: dict, timeout: int = 30) -> bool:
+        """Wait for page to refresh and stabilize"""
+        try:
+            start_time = time.time()
+            while time.time() - start_time < timeout:
+                current_state = self.get_page_state()
+                
+                # Check if page has reloaded
+                if (current_state['url'] == original_state['url'] and
+                    current_state['title'] == original_state['title']):
+                    
+                    # Check if DOM has finished updating
+                    is_stable = self.driver.execute_script("""
+                        return (typeof window._lastMutation === 'undefined' || 
+                                Date.now() - window._lastMutation > 1000);
+                    """)
+                    
+                    if is_stable:
+                        return True
+                        
+                time.sleep(0.1)
+            
+            self.log("Timeout waiting for page to stabilize")
+            return False
+            
+        except Exception as e:
+            self.log(f"Error waiting for page change: {str(e)}")
+            return False
+
+    def setup_mutation_tracking(self):
+        """Setup mutation observer to track DOM changes"""
+        self.driver.execute_script("""
+            // Track last mutation time
+            window._lastMutation = Date.now();
+            
+            // Create mutation observer
+            const observer = new MutationObserver(() => {
+                window._lastMutation = Date.now();
+            });
+            
+            // Observe entire document for any changes
+            observer.observe(document, {
+                childList: true,
+                subtree: true,
+                attributes: true,
+                characterData: true
+            });
+        """)
+
+    def refresh(self, hard: bool = False, timeout: int = 30) -> bool:
+        """
+        Refresh the current page
+        
+        Args:
+            hard (bool): If True, bypass cache with Ctrl+F5
+            timeout (int): Maximum time to wait for refresh in seconds
+            
+        Returns:
+            bool: True if refresh was successful
+        """
+        try:
+            # Get initial page state
+            original_state = self.get_page_state()
+            self.log(f"Original page state: {original_state}")
+            
+            # Setup mutation tracking
+            self.setup_mutation_tracking()
+            
+            if hard:
+                # Force reload bypassing cache
+                self.log("Executing hard refresh (Ctrl+F5)")
+                self.driver.execute_script("""
+                    window.location.reload(true);
+                """)
+            else:
+                # Normal reload
+                self.log("Executing normal refresh")
+                self.driver.refresh()
+            
+            # Wait for page to stabilize
+            success = self.wait_for_page_change(original_state, timeout)
+            
+            if success:
+                # Restore scroll position
+                self.driver.execute_script(
+                    "window.scrollTo(arguments[0], arguments[1]);",
+                    original_state['scroll_position']['scroll_x'],
+                    original_state['scroll_position']['scroll_y']
+                )
+                
+                # Final verification
+                new_state = self.get_page_state()
+                self.log(f"New page state: {new_state}")
+                
+                return True
+            
+            return False
+            
+        except Exception as e:
+            self.log(f"Refresh failed: {str(e)}")
+            return False
+
+def main():
+    parser = argparse.ArgumentParser(description='Refresh page in Chrome')
+    parser.add_argument('debug_port', type=int, help='Chrome debug port')
+    parser.add_argument('--hard', action='store_true', help='Force refresh bypassing cache')
+    parser.add_argument('--timeout', type=int, default=30, help='Refresh timeout in seconds')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Enable debug output')
+    
+    args = parser.parse_args()
+    
+    try:
+        controller = RefreshController(args.debug_port, args.verbose)
+        success = controller.refresh(args.hard, args.timeout)
+        sys.exit(0 if success else 1)
+        
+    except Exception as e:
+        if args.verbose:
+            print(f"Fatal error: {str(e)}", file=sys.stderr)
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

tools/itb/bin/itb_screenshot

@@ -1,12 +1,15 @@
 #!/usr/bin/env python3
 import sys
 import os
+import argparse
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 import xml.etree.ElementTree as ET
 
 class ScreenshotGenerator:
-    def __init__(self, debug_port: int):
+    def __init__(self, debug_port: int, debug: bool = False):
+        """Initialize the screenshot generator with optional debug mode"""
+        self.debug = debug
         options = webdriver.ChromeOptions()
         options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
         self.driver = webdriver.Chrome(options=options)
@@ -18,69 +21,294 @@ class ScreenshotGenerator:
             with open(os.path.join(js_dir, filename)) as f:
                 self.js_code += f.read() + "\n"
 
-    def get_page_content(self) -> dict:
-        return self.driver.execute_script(self.js_code + "; return analyzePageContent();")
+    def log(self, msg: str):
+        """Log message if verbose mode is enabled"""
+        if self.debug:
+            print(f"[Debug] {msg}")
 
-    def format_xml(self, xml_str: str) -> str:
-        root = ET.fromstring(xml_str)
-        ET.indent(ET.ElementTree(root), space="  ", level=0)
-        return ET.tostring(root, encoding='unicode', method='xml')
+    def get_visible_dom(self) -> str:
+        """Get only the visible DOM elements with enhanced visibility detection"""
+        if self.debug:
+            self.log("Starting DOM analysis")
+            print("Debug mode enabled, gathering DOM information...")
         
+        try:
+            script = """
+            // Debug helper
+            const debug = {
+                log: [],
+                add: function(msg) {
+                    this.log.push(msg);
+                }
+            };
+
+            function isVisible(element) {
+                debug.add(`Checking visibility for ${element.tagName}`);
+                
+                if (!element || element.nodeType !== Node.ELEMENT_NODE) {
+                    debug.add('Not an element node');
+                    return false;
+                }
+                
+                const style = window.getComputedStyle(element);
+                const rect = element.getBoundingClientRect();
+                
+                debug.add(`Element: ${element.tagName}, display: ${style.display}, visibility: ${style.visibility}`);
+                debug.add(`Dimensions: ${rect.width}x${rect.height}`);
+                
+                return (
+                    style.display !== 'none' &&
+                    style.visibility !== 'hidden' &&
+                    rect.width > 0 &&
+                    rect.height > 0
+                );
+            }
+
+            function getElementInfo(element) {
+                const info = {
+                    tag: element.tagName.toLowerCase(),
+                    attributes: {},
+                    text: element.textContent.trim()
+                };
+                
+                ['id', 'class', 'href', 'src'].forEach(attr => {
+                    if (element.hasAttribute(attr)) {
+                        info.attributes[attr] = element.getAttribute(attr);
+                    }
+                });
+                
+                return info;
+            }
+
+            function traverseDOM(element, depth = 0) {
+                if (!element) {
+                    debug.add('Null element encountered');
+                    return '';
+                }
+
+                let output = '';
+                if (element.nodeType === Node.ELEMENT_NODE && isVisible(element)) {
+                    const info = getElementInfo(element);
+                    const indent = '  '.repeat(depth);
+                    
+                    let elementStr = `${indent}${info.tag}`;
+                    Object.entries(info.attributes).forEach(([key, value]) => {
+                        elementStr += ` ${key}="${value}"`;
+                    });
+                    
+                    if (info.text && !element.children.length) {
+                        elementStr += ` [text: ${info.text}]`;
+                    }
+                    
+                    output += elementStr + '\\n';
+                    
+                    Array.from(element.children).forEach(child => {
+                        output += traverseDOM(child, depth + 1);
+                    });
+                }
+                
+                return output;
+            }
+
+            const dom = traverseDOM(document.documentElement);
+            debug.add(`Total output length: ${dom.length}`);
+
+            return {
+                result: dom,
+                debug: debug.log,
+                stats: {
+                    url: window.location.href,
+                    title: document.title,
+                    elementCount: document.getElementsByTagName('*').length,
+                    bodyChildCount: document.body ? document.body.children.length : 0
+                }
+            };
+            """
+            
+            result = self.driver.execute_script(script)
+            
+            if isinstance(result, dict):
+                if self.debug:
+                    print("\n=== Page Statistics ===")
+                    stats = result.get('stats', {})
+                    print(f"Title: {stats.get('title')}")
+                    print(f"Element count: {stats.get('elementCount')}")
+                    print(f"Body children: {stats.get('bodyChildCount')}")
+                    print("\n=== Debug Log ===")
+                    for log_entry in result.get('debug', []):
+                        print(f"Browser: {log_entry}")
+                
+                return result.get('result', '')
+            return result
+            
+        except Exception as e:
+            if self.debug:
+                print(f"Error in get_visible_dom: {str(e)}")
+            return ""
+
+    def get_full_dom(self) -> str:
+        """Get the full DOM structure"""
+        return self.driver.execute_script("""
+            function traverseDOM(node, indent = '') {
+                if (!node) return '';
+                let result = '';
+                
+                // Add node information
+                if (node.nodeType === Node.ELEMENT_NODE) {
+                    result += indent + node.tagName.toLowerCase();
+                    if (node.id) result += ` id="${node.id}"`;
+                    if (node.className) result += ` class="${node.className}"`;
+                    result += '\\n';
+                    
+                    // Process children
+                    for (let child of node.children) {
+                        result += traverseDOM(child, indent + '  ');
+                    }
+                }
+                return result;
+            }
+            return traverseDOM(document.documentElement);
+        """)
+
     def generate_xml(self) -> str:
-        page_data = self.get_page_content()
+        page_data = self.driver.execute_script(self.js_code + "; return analyzePageContent();")
         
+        # Create viewport element with attributes
         viewport = ET.Element("viewport")
-        for k, v in page_data['viewport'].items():
-            viewport.set(k, str(v))
+        for key, value in page_data['viewport'].items():
+            viewport.set(key, str(value))
         
-        def process_node(node, parent):
-            if node is None:
-                return
-                
-            if node['text']:
-                if len(parent) == 0:
-                    parent.text = (parent.text or '') + node['text'] + '\n'
+        def process_element(element_data, parent):
+            # Add fixed position indication if present
+            if element_data.get('fixed'):
+                if element_data.get('tag') == 'nav':
+                    nav = ET.SubElement(parent, "navigation")
+                    nav.set("fixed", "true")
+                    nav.set("x", str(element_data['x']))
+                    nav.set("y", str(element_data['y']))
+                    nav.set("width", str(element_data['width']))
+                    nav.set("height", str(element_data['height']))
+                    current_parent = nav
                 else:
-                    parent[-1].tail = (parent[-1].tail or '') + node['text'] + '\n'
-                    
-            if node['interactions']:
-                interactive = ET.SubElement(parent, "interactive")
-                interactive.set("x", str(node['location']['x']))
-                interactive.set("y", str(node['location']['y']))
-                interactive.set("width", str(node['size']['width']))
-                interactive.set("height", str(node['size']['height']))
-                interactive.set("interactions", ",".join(
-                    i for i in node['interactions'] if not i.startswith('placeholder:')
-                ))
+                    fixed_container = ET.SubElement(parent, "fixed")
+                    fixed_container.set("x", str(element_data['x']))
+                    fixed_container.set("y", str(element_data['y']))
+                    fixed_container.set("width", str(element_data['width']))
+                    fixed_container.set("height", str(element_data['height']))
+                    current_parent = fixed_container
+            else:
+                current_parent = parent
+
+            # Handle text content
+            if element_data.get('text'):
+                if len(current_parent) == 0:
+                    current_parent.text = (current_parent.text or '') + element_data['text'] + '\n'
+                else:
+                    current_parent[-1].tail = (current_parent[-1].tail or '') + element_data['text'] + '\n'
+            
+            # Handle interactive elements
+            if element_data.get('interactions'):
+                interactive = ET.SubElement(current_parent, "interactive")
+                interactive.set("x", str(element_data['x']))
+                interactive.set("y", str(element_data['y']))
+                interactive.set("width", str(element_data['width']))
+                interactive.set("height", str(element_data['height']))
+                interactive.set("interactions", ','.join(element_data['interactions']))
                 
-                placeholder = next(
-                    (i.split(':', 1)[1] for i in node['interactions'] 
-                     if i.startswith('placeholder:')),
-                    None
-                )
-                if placeholder:
-                    interactive.set('placeholder', placeholder)
-                    
-                if node['text']:
-                    interactive.text = node['text']
-                    
-            for child in node['children']:
-                process_node(child, parent)
+                if element_data.get('id'):
+                    interactive.set("id", element_data['id'])
+                elif element_data.get('suggestedClick'):
+                    interactive.set("suggested_click", 
+                        f"{element_data['suggestedClick']['x']},{element_data['suggestedClick']['y']}")
+                
+                if element_data.get('placeholder'):
+                    interactive.set("placeholder", element_data['placeholder'])
+                
+                if element_data.get('buttonText'):
+                    interactive.text = element_data['buttonText']
+            
+            # Handle images
+            if element_data.get('tag') == 'img':
+                img = ET.SubElement(current_parent, "img")
+                img.set("x", str(element_data['x']))
+                img.set("y", str(element_data['y']))
+                img.set("width", str(element_data['width']))
+                img.set("height", str(element_data['height']))
+                img.set("src", element_data.get('src', ''))
+                img.set("alt", element_data.get('alt', ''))
+                if element_data.get('id'):
+                    img.set("id", element_data['id'])
+
+            # Process children
+            for child in element_data.get('children', []):
+                process_element(child, current_parent)
+        
+        # Process all elements
+        process_element(page_data['content'], viewport)
         
-        process_node(page_data['content'], viewport)
         return self.format_xml(ET.tostring(viewport, encoding='unicode'))
 
-def main(debug_port: int):
+    def format_xml(self, xml_str: str) -> str:
+        """Format XML with proper indentation"""
+        root = ET.fromstring(xml_str)
+        ET.indent(root, space="  ", level=0)
+        return ET.tostring(root, encoding='unicode', method='xml')
+
+def main():
+    parser = argparse.ArgumentParser(description='Generate page screenshot in XML format')
+    parser.add_argument('debug_port', type=int, help='Chrome debug port')
+    parser.add_argument('--debug', action='store_true', help='Enable debug output')
+    parser.add_argument('--dom', action='store_true', help='Print full DOM structure')
+    parser.add_argument('--visible-dom', action='store_true', help='Print visible DOM structure')
+    
+    args = parser.parse_args()
+    
     try:
-        generator = ScreenshotGenerator(debug_port)
-        xml_output = generator.generate_xml()
-        print(xml_output)
+        generator = ScreenshotGenerator(args.debug_port, args.debug)
+        
+        # Basic DOM check when debug is enabled
+        if args.debug:
+            print("\nPerforming basic DOM check...")
+            basic_check = generator.driver.execute_script("""
+                return {
+                    title: document.title,
+                    bodyExists: !!document.body,
+                    elementCount: document.getElementsByTagName('*').length,
+                    firstElement: document.body ? document.body.firstElementChild.tagName : 'none'
+                }
+            """)
+            print(f"Page title: {basic_check['title']}")
+            print(f"Body exists: {basic_check['bodyExists']}")
+            print(f"Total elements: {basic_check['elementCount']}")
+            print(f"First element: {basic_check['firstElement']}")
+        
+        if args.dom:
+            print("=== Full DOM Structure ===")
+            try:
+                dom_output = generator.get_full_dom()
+                if dom_output.strip():
+                    print(dom_output)
+                else:
+                    print("No DOM content found.")
+            except Exception as e:
+                print(f"Error retrieving DOM structure: {str(e)}")
+        elif args.visible_dom:
+            print("=== Visible DOM Structure ===")
+            try:
+                visible_dom = generator.get_visible_dom()
+                if visible_dom.strip():
+                    print(visible_dom)
+                else:
+                    print("No visible DOM content found.")
+            except Exception as e:
+                print(f"Error retrieving visible DOM structure: {str(e)}")
+        else:
+            xml_output = generator.generate_xml()
+            print(xml_output)
+            
     except Exception as e:
         print(f"Error: {e}", file=sys.stderr)
         sys.exit(1)
 
 if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: itb_screenshot <debug_port>")
-        sys.exit(1)
-    main(int(sys.argv[1]))
+    main()

tools/itb/example.md

@@ -1,267 +0,0 @@
-# Comprehensive ITB Social Media Interaction Example
-
-## Overview
-
-This example demonstrates how an AI agent uses ITB to interact with a social media interface. It showcases ITB's key features including text collapsing, state management, and complex interactions. The scenario walks through a complete social media interaction flow, from content creation to user engagement.
-
-## User Request to SIA
-
-"I want you to help me create a social media post about my morning hike. I have a photo at '/photos/sunrise.jpg' to share. First, search for similar hiking posts to see what hashtags are popular. Then create a new post with my photo and some engaging text. After posting, find and interact with other hiking-related posts - like them and leave encouraging comments. Finally, send a chat message to anyone who's online and interested in hiking."
-
-## SIA's Response and Actions
-
-### Step 1: Initialize Session
-
-```xml
-<reasoning>
-First, I'll start an ITB session and establish continuous monitoring of the page state.
-</reasoning>
-
-<single>
-itb_start "social.html"
-</single>
-```
-
-Expected Output:
-```json
-{
-    "pid": 12345,
-    "session_id": "550e8400-e29b-41d4-a716-446655440000",
-    "status": "success"
-}
-```
-
-```xml
-<repeat>
-itb_screenshot 550e8400-e29b-41d4-a716-446655440000 --wait_stable
-</repeat>
-```
-
-Initial viewport state:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="0">
-    <!-- Navigation bar with collapsed text -->
-    <text x="20" y="30">SocialConnect | Search posts... | Messages | Notifications</text>
-    
-    <!-- Interactive navigation elements -->
-    <interactive accepts_text="true" x="200" y="30" width="300" height="30" 
-                 placeholder="Search posts..." aria-label="Search posts"/>
-    <interactive clickable="true" id="profile_menu" x="720" y="30" width="30" height="30">
-        <img src="https://picsum.photos/100" alt="Your profile picture"/>
-    </interactive>
-    
-    <!-- Post creation area -->
-    <interactive accepts_text="true" x="340" y="100" width="400" height="100"
-                 placeholder="What's on your mind?" aria-label="Create a post"/>
-    <interactive accepts_file="true" id="post_images" x="340" y="130" width="200" height="30"
-                 multiple="true" accept="image/*"/>
-    <interactive clickable="true" x="340" y="160" width="80" height="30">Post</interactive>
-
-    <!-- Feed posts with collapsed text content -->
-    <text x="100" y="220">Alice Johnson - 2 hours ago - Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness</text>
-    <img src="https://picsum.photos/800/600?random=2" alt="Sunrise view from mountain top" x="100" y="320" width="600" height="450"/>
-    
-    <!-- Interactive post actions -->
-    <interactive clickable="true" x="340" y="520" width="80" height="30">Like</interactive>
-    <interactive clickable="true" x="340" y="560" width="80" height="30">Comment</interactive>
-    <interactive clickable="true" x="340" y="600" width="80" height="30">Share</interactive>
-
-    <!-- Sidebar with collapsed text -->
-    <text x="960" y="30">Online Friends</text>
-    <interactive clickable="true" x="960" y="200" width="200" height="40">Emma Davis (Online)</interactive>
-</viewport>
-```
-
-### Step 2: Search for Hiking Posts
-
-```xml
-<reasoning>
-I see the search box in the navigation bar. I'll click it and search for hiking-related posts to analyze common hashtags.
-</reasoning>
-
-<single>
-itb_cursor 550e8400-e29b-41d4-a716-446655440000 -x 250 -y 30 --click
-</single>
-
-<single>
-itb_input 550e8400-e29b-41d4-a716-446655440000 -t "#hiking"
-</single>
-```
-
-Updated viewport showing search results:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="0">
-    <!-- Navigation and search UI remain unchanged -->
-    
-    <!-- Search results header -->
-    <text x="100" y="80">Search results for "#hiking" - 3 posts found</text>
-    
-    <!-- Collapsed search result posts -->
-    <text x="100" y="120">Alice Johnson - 2 hours ago - Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness</text>
-    <img src="https://picsum.photos/800/600?random=2" alt="Sunrise view from mountain top" x="100" y="220" width="600" height="450"/>
-    
-    <text x="100" y="720">David Brown - 1 day ago - Weekend hiking adventure! Found this hidden waterfall trail. #Hiking #Adventure #NaturePhotography</text>
-    <img src="https://picsum.photos/800/600?random=3" alt="Waterfall on hiking trail" x="100" y="820" width="600" height="450"/>
-</viewport>
-```
-
-### Step 3: Create New Post
-
-```xml
-<reasoning>
-Having analyzed the existing posts and popular hashtags, I'll now create our new post with the sunrise photo.
-</reasoning>
-
-<single>
-itb_scroll 550e8400-e29b-41d4-a716-446655440000 -x 0 -y 0
-itb_cursor 550e8400-e29b-41d4-a716-446655440000 -x 340 -y 100 --click
-</single>
-
-<single>
-itb_input 550e8400-e29b-41d4-a716-446655440000 -t "Starting my day with an invigorating mountain hike! The colors of dawn painted the sky in stunning shades of orange and purple. Nothing beats the serenity of nature in the early morning. 🌄 #MorningHike #Nature #Wellness #SunriseViews"
-</single>
-
-<single>
-itb_upload 550e8400-e29b-41d4-a716-446655440000 -i "post_images" -f "/photos/sunrise.jpg"
-</single>
-```
-
-Viewport showing post creation state:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="0">
-    <!-- Navigation and header remain unchanged -->
-    
-    <!-- Post creation area with entered content -->
-    <text x="340" y="100">Starting my day with an invigorating mountain hike! The colors of dawn painted the sky in stunning shades of orange and purple. Nothing beats the serenity of nature in the early morning. 🌄 #MorningHike #Nature #Wellness #SunriseViews</text>
-    <img src="file:///photos/sunrise.jpg" alt="Uploaded sunrise photo" x="340" y="220" width="400" height="300"/>
-    <interactive clickable="true" x="720" y="220" width="20" height="20" aria-label="Remove uploaded photo">×</interactive>
-</viewport>
-```
-
-```xml
-<single>
-itb_cursor 550e8400-e29b-41d4-a716-446655440000 -x 340 -y 160 --click
-</single>
-```
-
-Viewport after post creation:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="0">
-    <!-- Navigation remains unchanged -->
-    
-    <!-- Newly created post appears at top of feed -->
-    <text x="100" y="120">You - Just now - Starting my day with an invigorating mountain hike! The colors of dawn painted the sky in stunning shades of orange and purple. Nothing beats the serenity of nature in the early morning. 🌄 #MorningHike #Nature #Wellness #SunriseViews</text>
-    <img src="file:///photos/sunrise.jpg" alt="Uploaded sunrise photo" x="100" y="220" width="600" height="450"/>
-</viewport>
-```
-
-### Step 4: Interact with Related Posts
-
-```xml
-<reasoning>
-Now I'll scroll to find Alice's hiking post and engage with it through likes and comments.
-</reasoning>
-
-<single>
-itb_scroll 550e8400-e29b-41d4-a716-446655440000 --to-element "post-actions"
-</single>
-
-<single>
-itb_cursor 550e8400-e29b-41d4-a716-446655440000 -x 340 -y 520 --click
-</single>
-```
-
-Viewport after liking:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="600">
-    <text x="100" y="120">Alice Johnson - 2 hours ago - Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness</text>
-    <img src="https://picsum.photos/800/600?random=2" alt="Sunrise view from mountain top" x="100" y="220" width="600" height="450"/>
-    <interactive clickable="true" x="340" y="520" width="80" height="30" state="active">Like</interactive>
-</viewport>
-```
-
-```xml
-<single>
-itb_cursor 550e8400-e29b-41d4-a716-446655440000 -x 340 -y 580 --click
-itb_input 550e8400-e29b-41d4-a716-446655440000 -t "What a gorgeous view! I just did a morning hike too. The sunrise is always worth the early wake-up call! 🌅"
-</single>
-```
-
-### Step 5: Start a Chat
-
-```xml
-<reasoning>
-I notice Emma Davis is online and has hiking photos. I'll initiate a conversation about hiking.
-</reasoning>
-
-<single>
-itb_cursor 550e8400-e29b-41d4-a716-446655440000 -x 960 -y 200 --click
-</single>
-```
-
-Viewport showing chat window:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="600">
-    <!-- Main content remains with added chat widget -->
-    <interactive class="chat-widget" x="744" y="400" width="280" height="400">
-        <text x="754" y="420">Emma Davis (Online)</text>
-        <interactive accepts_text="true" x="754" y="760" width="260" height="30"
-                     placeholder="Type a message..." aria-label="Type a message"/>
-    </interactive>
-</viewport>
-```
-
-```xml
-<single>
-itb_input 550e8400-e29b-41d4-a716-446655440000 -t "Hi Emma! I noticed you're into hiking too. Would you be interested in joining for a sunrise hike sometime? I just discovered some beautiful trails!"
-</single>
-```
-
-Final viewport state:
-```xml
-<viewport width="1024" height="800" scroll_x="0" scroll_y="600">
-    <!-- Chat widget with sent message -->
-    <interactive class="chat-widget" x="744" y="400" width="280" height="400">
-        <text x="754" y="420">Emma Davis (Online)</text>
-        <text x="754" y="480">You - Just now - Hi Emma! I noticed you're into hiking too. Would you be interested in joining for a sunrise hike sometime? I just discovered some beautiful trails!</text>
-        <interactive accepts_text="true" x="754" y="760" width="260" height="30"
-                     placeholder="Type a message..." aria-label="Type a message"/>
-    </interactive>
-</viewport>
-```
-
-```xml
-<reasoning>
-I've completed all requested interactions:
-1. Analyzed existing hiking posts and hashtags
-2. Created a new post with photo and appropriate hashtags
-3. Engaged with Alice's hiking post through a like and comment
-4. Initiated a conversation with Emma about hiking
-The continuous screenshot monitoring remains active to catch any responses.
-</reasoning>
-```
-
-## Key Features Demonstrated
-
-This example illustrates several important ITB features:
-
-1. Text Collapsing: Related text elements are automatically combined into natural reading flows, such as "Author - Time - Content" for posts.
-
-2. State Management: The viewport maintains awareness of scroll position, focus state, and interactive element states throughout the session.
-
-3. Interactive Elements: The example shows how ITB handles various types of interaction including:
-   - Text input fields
-   - File uploads
-   - Clickable buttons
-   - Scrollable areas
-   - Modal windows (chat widget)
-
-4. Media Handling: Images are represented with their position, size, and alt text while maintaining proper layout relationships with surrounding content.
-
-5. Dynamic Content: The example demonstrates how ITB handles dynamic content updates like:
-   - Search results loading
-   - Post creation
-   - Chat window opening
-   - Comment threading
-
-This comprehensive example shows how ITB transforms complex web interfaces into a format that AI agents can effectively process and interact with while maintaining all the functionality a human user would have.

tools/itb/js/dom_analyzer.js

@@ -1,67 +1,208 @@
-function processElement(element) {
-    const rect = element.getBoundingClientRect();
-    if (!element.offsetParent && element !== document.body) return null;
-    if (rect.width === 0 || rect.height === 0) return null;
-    
-    const viewportWidth = window.innerWidth;
-    const viewportHeight = window.innerHeight;
-    if (rect.bottom < 0 || rect.top > viewportHeight || 
-        rect.right < 0 || rect.left > viewportWidth) {
-        return null;
+// Helper function to check if element is clickable
+function isClickable(element) {
+    // Check tag name
+    if (element.tagName === 'BUTTON' || 
+        element.tagName === 'A' || 
+        (element.tagName === 'INPUT' && element.type === 'submit')) {
+        return true;
     }
     
-    const styles = window.getComputedStyle(element);
+    // Check ARIA role
+    if (element.getAttribute('role') === 'button') {
+        return true;
+    }
     
-    const text = Array.from(element.childNodes)
-        .filter(node => node.nodeType === Node.TEXT_NODE)
-        .map(node => node.textContent.trim())
-        .join(' ');
-        
-    const interactions = [];
-    if (element.tagName === 'BUTTON' || element.tagName === 'A' || 
-        element.getAttribute('role') === 'button' || 
+    // Check for click handlers
+    if (element.onclick || 
         element.getAttribute('onclick') || 
-        styles.cursor === 'pointer') {
-        interactions.push('click');
+        element.getAttribute('ng-click') ||
+        element.getAttribute('v-on:click')) {
+        return true;
     }
     
-    if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA' ||
-        element.getAttribute('contenteditable') === 'true') {
-        interactions.push('text_input');
-        const placeholder = element.getAttribute('placeholder');
-        if (placeholder) interactions.push('placeholder:' + placeholder);
+    // Check computed style
+    const style = window.getComputedStyle(element);
+    if (style.cursor === 'pointer') {
+        return true;
     }
     
-    if (element.tagName === 'SELECT' || 
-        element.getAttribute('role') === 'listbox' ||
-        element.getAttribute('type') === 'checkbox' ||
-        element.getAttribute('type') === 'radio') {
-        interactions.push('select');
+    return false;
+}
+
+// Helper function to check if element is an input
+function isInputElement(element) {
+    const inputTags = ['INPUT', 'TEXTAREA', 'SELECT'];
+    if (inputTags.includes(element.tagName)) {
+        return true;
     }
     
-    if (element.scrollHeight > element.clientHeight || 
-        element.scrollWidth > element.clientWidth) {
-        interactions.push('scroll');
+    if (element.getAttribute('contenteditable') === 'true') {
+        return true;
     }
     
-    const children = Array.from(element.children)
-        .map(child => processElement(child))
-        .filter(child => child !== null);
+    if (element.getAttribute('role') === 'textbox' ||
+        element.getAttribute('role') === 'combobox') {
+        return true;
+    }
+    
+    return false;
+}
+
+// Helper function to check if element is scrollable
+function isScrollable(element) {
+    const style = window.getComputedStyle(element);
+    
+    // Check for scroll overflow
+    if (style.overflow === 'auto' || 
+        style.overflow === 'scroll' ||
+        style.overflowY === 'auto' || 
+        style.overflowY === 'scroll' ||
+        style.overflowX === 'auto' || 
+        style.overflowX === 'scroll') {
         
-    return {
+        // Verify element has scrollable content
+        return element.scrollHeight > element.clientHeight ||
+               element.scrollWidth > element.clientWidth;
+    }
+    
+    return false;
+}
+
+// Helper function to check if element is actually visible
+function isElementVisible(element) {
+    const style = window.getComputedStyle(element);
+    
+    // Special handling for fixed position elements
+    if (style.position === 'fixed') {
+        // Fixed elements are always considered visible if they have dimensions
+        if (element.offsetWidth > 0 && element.offsetHeight > 0) {
+            return true;
+        }
+    }
+    
+    // For non-fixed elements, check offsetParent
+    if (!element.offsetParent && element !== document.body) {
+        return false;
+    }
+    
+    if (style.display === 'none' || 
+        style.visibility === 'hidden' || 
+        style.opacity === '0') {
+        return false;
+    }
+    
+    const rect = element.getBoundingClientRect();
+    if (rect.width === 0 || rect.height === 0) {
+        return false;
+    }
+    
+    // For non-fixed elements, check viewport boundaries
+    if (style.position !== 'fixed') {
+        const viewportWidth = window.innerWidth;
+        const viewportHeight = window.innerHeight;
+        
+        if (rect.right < 0 || 
+            rect.bottom < 0 || 
+            rect.left > viewportWidth || 
+            rect.top > viewportHeight) {
+            return false;
+        }
+    }
+    
+    return true;
+}
+
+// Helper function to get element text
+function getElementText(element) {
+    // Only process direct text nodes
+    const textNodes = Array.from(element.childNodes)
+        .filter(node => node.nodeType === Node.TEXT_NODE);
+        
+    if (textNodes.length === 0) return '';
+    
+    return textNodes
+        .map(node => node.textContent.trim())
+        .filter(text => text.length > 0)
+        .join(' ');
+}
+
+function processElements(element) {
+    if (!element || !isElementVisible(element)) return [];
+    
+    const rect = element.getBoundingClientRect();
+    const styles = window.getComputedStyle(element);
+    const isFixed = styles.position === 'fixed';
+    
+    // Get element base info
+    const elementInfo = {
         tag: element.tagName.toLowerCase(),
-        text,
-        location: {x: Math.round(rect.left), y: Math.round(rect.top)},
-        size: {width: Math.round(rect.width), height: Math.round(rect.height)},
-        interactions,
-        children
+        x: Math.round(isFixed ? rect.left : rect.left + window.pageXOffset),
+        y: Math.round(isFixed ? rect.top : rect.top + window.pageYOffset),
+        width: Math.round(rect.width),
+        height: Math.round(rect.height),
+        interactions: [],
+        fixed: isFixed
     };
+    
+    // Add ID or suggested click coordinates for any interactive element
+    if (isClickable(element) || isInputElement(element)) {
+        const id = element.id;
+        if (id) {
+            elementInfo.id = id;
+        } else {
+            // Only add suggested click if there's no ID
+            elementInfo.suggestedClick = {
+                x: Math.round(elementInfo.x + elementInfo.width / 2),
+                y: Math.round(elementInfo.y + elementInfo.height / 2)
+            };
+        }
+    }
+    
+    // Add text only if this isn't going to be an interactive element
+    const isInteractive = isClickable(element) || isInputElement(element);
+    if (!isInteractive) {
+        elementInfo.text = getElementText(element);
+    }
+    
+    // Add interactions
+    if (isClickable(element)) {
+        elementInfo.interactions.push('click');
+        elementInfo.buttonText = getElementText(element);
+    }
+    
+    if (isInputElement(element)) {
+        const inputType = element.getAttribute('type') || 'text';
+        elementInfo.interactions.push(`input:${inputType}`);
+        
+        const placeholder = element.getAttribute('placeholder');
+        if (placeholder) {
+            elementInfo.placeholder = placeholder;
+        }
+    }
+    
+    if (isScrollable(element)) {
+        elementInfo.interactions.push('scroll');
+    }
+    
+    // Special handling for images
+    if (element.tagName === 'IMG') {
+        elementInfo.src = element.src;
+        elementInfo.alt = element.alt;
+    }
+    
+    // Process children
+    elementInfo.children = Array.from(element.children)
+        .map(child => processElements(child))
+        .flat()
+        .filter(child => child !== null);
+    
+    return elementInfo;
 }
 
 function analyzePageContent() {
     const body = document.body;
     return {
         viewport: getViewportInfo(),
-        content: body ? processElement(body) : null
+        content: body ? processElements(body) : []
     };
-}
+}

tools/itb/result.social.html

@@ -1,19 +0,0 @@
-<viewport scroll_x="0" scroll_y="0" width="780" height="441" max_scroll_y="1200" max_scroll_x="0">
-    SocialConnect
-    <interactive x="16" y="0" width="240" height="40" interactions="text_input" placeholder="Search posts..."/>
-    Alice Johnson
-    2 hours ago
-    Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
-    <img x="36" y="220" width="648" height="400" alt="Sunrise view from mountain top" src="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' width='648' height='400'><rect width='100%' height='100%' fill='%23ddd'/></svg>"/>
-    <interactive x="36" y="632" width="216" height="40" interactions="click">
-        Like
-    </interactive>
-    <interactive x="252" y="632" width="216" height="40" interactions="click">
-        Comment
-    </interactive>
-    <interactive x="468" y="632" width="216" height="40" interactions="click">
-        Share
-    </interactive>
-    Bob Smith
-    Beautiful view! Which trail is this?
-</viewport>

tools/itb/result.test.html

@@ -1,32 +0,0 @@
-<viewport scroll_y="0" scroll_x="0" height="800" width="1024">
-  <header class="sticky" y="0" x="0" width="1024" height="50">
-    <form id="search">
-      <input type="text" placeholder="Search..." aria-label="Search box" y="10" x="10" width="200" height="30" />
-      <button type="submit" y="10" x="220" width="30" height="30">🔍</button>
-    </form>
-  </header>
-  
-  <section class="scrollable" y="50" x="0" width="1024" height="200" scroll_y="0" max_scroll="400">
-    <heading level="2">Scrollable Section</heading>
-    Lorem ipsum dolor sit amet, consectetur adipiscing elit... [first paragraph]
-  </section>
-
-  <interactive y="260" x="10" width="100" height="30">
-    <button onclick="showModal()">Show Modal</button>
-  </interactive>
-  
-  <interactive y="260" x="120" width="120" height="30">
-    <link href="#">JavaScript Link</link>
-  </interactive>
-  
-  <interactive y="260" x="250" width="100" height="30">
-    <button role="button" tabindex="0">ARIA Button</button>
-  </interactive>
-
-  <table y="300" x="0" width="1024">
-    <cell>Cell with mixed formatting</cell>
-    <cell>Cell with list:
-- Item 1
-- Item 2</cell>
-  </table>
-</viewport>

tools/itb/setup.py

@@ -10,8 +10,6 @@ setup(
         'bin/itb_scroll',
         'bin/itb_click',
         'bin/itb_input',
-        'bin/itb_links',
-        'bin/itb_forms',
         'bin/itb_navigate',
         'bin/itb_refresh'
     ],

tools/itb/test.html

@@ -1,90 +0,0 @@
-<!DOCTYPE html>
-<html>
-<head>
-    <style>
-        .scrollable-div {
-            height: 200px;
-            overflow-y: scroll;
-            border: 1px solid black;
-        }
-        .sticky-header {
-            position: sticky;
-            top: 0;
-            background: white;
-        }
-        .modal {
-            position: fixed;
-            top: 50%;
-            left: 50%;
-            transform: translate(-50%, -50%);
-            z-index: 100;
-        }
-    </style>
-</head>
-<body>
-    <!-- Sticky header with form -->
-    <header class="sticky-header">
-        <form id="search">
-            <input type="text" placeholder="Search..." aria-label="Search box">
-            <button type="submit">🔍</button>
-        </form>
-    </header>
-
-    <!-- Nested scrollable content -->
-    <div class="scrollable-div">
-        <h2>Scrollable Section</h2>
-        <p>
-Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam a bibendum purus, id accumsan lorem. Ut quis diam semper, rutrum justo in, ornare urna. Proin vel consectetur odio. Donec quis odio ut erat lobortis aliquam. Vestibulum vitae tellus interdum, hendrerit odio id, varius urna. Sed nec est tempor, placerat lectus sit amet, aliquam mauris. Duis elementum risus vitae nunc mollis ullamcorper. Donec sodales ac neque at aliquam. Duis eu risus ex. Fusce neque erat, egestas nec placerat in, dapibus quis magna. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Ut faucibus nunc tellus, vel consequat dolor suscipit vel. Sed sapien erat, rutrum non dignissim nec, egestas at augue. Nunc interdum lacinia urna a euismod. Integer ullamcorper sit amet sapien non auctor.
-        </p>
-        <p>
-Nam vitae bibendum nulla, ut tempus enim. Aenean eu iaculis metus, vel rhoncus justo. Sed vel interdum sem. Praesent congue erat sit amet ultrices molestie. Pellentesque pellentesque odio a ullamcorper facilisis. Sed at viverra magna. Aliquam posuere dolor tortor, non aliquam orci bibendum eu. Sed efficitur feugiat congue. Vivamus elementum in metus lacinia gravida.
-        </p>
-        <p>
-Fusce aliquam, erat ac consequat eleifend, massa neque posuere urna, vel sagittis tortor neque hendrerit augue. Nulla mattis felis non efficitur hendrerit. Pellentesque viverra elementum enim ornare pellentesque. Donec mollis eget leo non congue. Aliquam vel iaculis odio. Mauris eget neque suscipit, tempus mi quis, porta quam. Integer non tempor leo. Pellentesque magna nisl, fermentum sit amet orci ut, posuere dictum erat. Vivamus a lacinia lacus, eleifend ultrices velit. Etiam est arcu, dictum at sapien efficitur, fringilla tincidunt sem. In nec lacus quis diam placerat sollicitudin. Vivamus scelerisque commodo velit, id lacinia orci volutpat quis. Nulla facilisi. Nam nec quam quis lorem vulputate auctor quis a dui.
-        </p>
-        <p>
-Curabitur a erat nisi. Fusce lacinia, enim quis mollis placerat, eros nulla blandit turpis, id feugiat purus enim ut diam. Etiam et tincidunt turpis, in scelerisque lectus. Nunc vulputate at velit sit amet tincidunt. Cras maximus ultrices enim, sit amet placerat nisi tristique eget. Sed ac neque sed quam laoreet pellentesque nec vel lorem. Nunc sollicitudin arcu ac quam tempor bibendum. Phasellus sollicitudin tortor at porta lobortis. Aenean venenatis ultrices neque ut scelerisque. Nullam aliquam fringilla varius. Etiam eget nunc leo. Proin ornare tempus libero eget molestie. Praesent id ex sit amet ante semper dictum. Mauris ac libero at ipsum dapibus pretium quis at sapien. Praesent ac luctus lacus. Praesent id varius nisl, et luctus ex.
-        </p>
-        <iframe src="about:blank" title="Embedded content">
-            <p>
-Sed ullamcorper sapien non nunc venenatis rutrum. Nulla mollis eu augue non mollis. Maecenas molestie congue libero. Nulla non mi sem. Cras sed ipsum et orci porta posuere et in metus. Vivamus gravida leo a risus interdum, et aliquet neque pretium. Duis a mauris nec erat tempus lacinia. Cras a justo vel dui varius mattis sed ac neque. Phasellus blandit massa sed elit consectetur placerat. Sed aliquet et lacus nec posuere. Donec mattis dolor id tortor facilisis consectetur. Phasellus sit amet magna non nisl pretium semper. Sed viverra, mauris vitae ultricies interdum, libero felis aliquet massa, at vehicula elit lacus a diam. Aliquam sed erat et justo tempor mattis eget et neque. Ut fermentum magna ut lorem sodales condimentum. Vestibulum elementum nec urna sed suscipit. 
-            </p>
-        </iframe>
-        <div contenteditable="true">
-Curabitur a erat nisi. Fusce lacinia, enim quis mollis placerat, eros nulla blandit turpis, id feugiat purus enim ut diam. Etiam et tincidunt turpis, in scelerisque lectus. Nunc vulputate at velit sit amet tincidunt. Cras maximus ultrices enim, sit amet placerat nisi tristique eget. Sed ac neque sed quam laoreet pellentesque nec vel lorem. Nunc sollicitudin arcu ac quam tempor bibendum. Phasellus sollicitudin tortor at porta lobortis. Aenean venenatis ultrices neque ut scelerisque. Nullam aliquam fringilla varius. Etiam eget nunc leo. Proin ornare tempus libero eget molestie. Praesent id ex sit amet ante semper dictum. Mauris ac libero at ipsum dapibus pretium quis at sapien. Praesent ac luctus lacus. Praesent id varius nisl, et luctus ex.
-        </div>
-    </div>
-
-    <!-- Interactive elements with dynamic content -->
-    <div>
-        <button onclick="showModal()">Show Modal</button>
-        <a href="#" onclick="return false;">JavaScript Link</a>
-        <div role="button" tabindex="0">ARIA Button</div>
-    </div>
-
-    <!-- Modal with overlay -->
-    <div class="modal" style="display:none">
-        <h3>Modal Title</h3>
-        <form>
-            <select multiple>
-                <option>Choice 1</option>
-                <option>Choice 2</option>
-            </select>
-            <input type="range" min="0" max="100">
-        </form>
-    </div>
-
-    <!-- Complex text layout -->
-    <table>
-        <tr>
-            <td>Cell with <strong>mixed</strong> <em>formatting</em></td>
-            <td>Cell with list:
-                <ul>
-                    <li>Item 1</li>
-                    <li>Item 2</li>
-                </ul>
-            </td>
-        </tr>
-    </table>
-</body>
-</html>