ITB WIP

container.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 container_id=$(docker ps -q)
-docker exec -it $container_id /bin/bash
+docker exec -it $container_id bash

run.sh

@@ -25,5 +25,6 @@ docker run \
     -v /$(pwd)/model/:/root/model/ \
     -v /$(pwd)/.env:/root/.env \
     -v /$(pwd)/iterations/:/root/iterations/ \
+    -v /$(pwd)/tools/:/root/sia/tools/ \
     -v /$(pwd)/.git/:/root/sia/.git/ \
     sia "$@"

tools/itb/bin/itb_screenshot

@@ -4,174 +4,194 @@ from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.remote.webelement import WebElement
 import xml.etree.ElementTree as ET
-from typing import Dict, List, Set, Optional
+from typing import Dict, List, Set, Optional, Union
 from dataclasses import dataclass
 
 @dataclass
 class DOMNode:
     element: WebElement
     children: List['DOMNode']
-    own_text: str
+    text: str
-    total_text: str
     tag: str
     location: Dict[str, int]
     size: Dict[str, int]
-    is_interactive: bool
+    interactions: List[str]
-    attributes: Dict[str, str]
+    
+    def get_content(self) -> List[Union[ET.Element, str]]:
+        content = []
+        
+        # Add text as plain string if present 
+        if self.text:
+            content.append(self.text)
+            
+        # Add interactive element if present
+        if self.interactions:
+            attrs = {
+                "x": str(self.location["x"]),
+                "y": str(self.location["y"]), 
+                "width": str(self.size["width"]),
+                "height": str(self.size["height"])
+            }
+            
+            # Split interactions and placeholders
+            base_interactions = []
+            placeholder = None
+            
+            for interaction in self.interactions:
+                if interaction.startswith("placeholder:"):
+                    placeholder = interaction.split(":", 1)[1]
+                else:
+                    base_interactions.append(interaction)
+                    
+            attrs["interactions"] = ",".join(base_interactions)
+            if placeholder:
+                attrs["placeholder"] = placeholder
+                
+            interactive = ET.Element("interactive", attrs)
+            if self.text:
+                interactive.text = self.text
+            content.append(interactive)
+            
+        # Add all children's content
+        for child in self.children:
+            content.extend(child.get_content())
+            
+        return content
 
 class DOMTree:
     def __init__(self, root_element: WebElement):
+        print(f"\nBuilding DOM tree for {root_element.tag_name}")
         self.root = self._build_tree(root_element)
         
-    def _get_element_own_text(self, element: WebElement) -> str:
+    def _get_element_text(self, element: WebElement) -> str:
         script = """
             let element = arguments[0];
-            let walker = document.createTreeWalker(
-                element,
-                NodeFilter.SHOW_TEXT,
-                {
-                    acceptNode: function(node) {
-                        return node.parentElement === element ? 
-                            NodeFilter.FILTER_ACCEPT : 
-                            NodeFilter.FILTER_REJECT;
-                    }
-                }
-            );
             let text = '';
-            let node;
+            for (let node of element.childNodes) {
-            while (node = walker.nextNode()) {
+                if (node.nodeType === Node.TEXT_NODE) {
-                text += node.textContent;
+                    text += node.textContent;
+                }
             }
             return text.trim();
         """
-        return element.parent.execute_script(script, element)
+        text = element.parent.execute_script(script, element)
+        if text:
+            print(f"Got text: {text[:50]}...")
+        return text
         
-    def _is_interactive(self, element: WebElement) -> bool:
+    def _get_interactions(self, element: WebElement) -> List[str]:
-        return (
+        interactions = []
-            element.tag_name in {"button", "a", "input", "select", "textarea"} or
-            element.get_attribute("role") in {"button", "link", "textbox"} or
-            element.get_attribute("onclick") is not None or
-            element.get_attribute("contenteditable") == "true"
-        )
         
-    def _get_attributes(self, element: WebElement) -> Dict[str, str]:
+        if (element.tag_name in {"button", "a"} or
-        attrs = {}
+            element.get_attribute("role") in {"button", "link"} or
-        for attr in ['placeholder', 'aria-label', 'alt', 'title', 'role']:
+            element.get_attribute("onclick") or
-            if value := element.get_attribute(attr):
+            element.value_of_css_property("cursor") == "pointer"):
-                attrs[attr] = value
+            interactions.append("click")
-        return attrs
+            
+        if (element.tag_name in {"input", "textarea"} or
+            element.get_attribute("contenteditable") == "true" or
+            element.get_attribute("role") == "textbox"):
+            interactions.append("text_input")
+            # Get placeholder if available
+            placeholder = element.get_attribute("placeholder")
+            if placeholder:
+                interactions.append(f"placeholder:{placeholder}")
+            
+        if (element.tag_name == "select" or
+            element.get_attribute("role") == "listbox" or
+            element.get_attribute("type") in {"checkbox", "radio"}):
+            interactions.append("select")
+            
+        script = """
+            let el = arguments[0];
+            return el.scrollHeight > el.clientHeight ||
+                   el.scrollWidth > el.clientWidth;
+        """
+        if element.parent.execute_script(script, element):
+            interactions.append("scroll")
+            
+        if interactions:
+            print(f"Found interactions for {element.tag_name}: {interactions}")
+        return interactions
         
     def _build_tree(self, element: WebElement) -> DOMNode:
-        children = [
+        print(f"\nProcessing <{element.tag_name}>")
-            self._build_tree(child) 
-            for child in element.find_elements(By.XPATH, "./*")
-        ]
-        
-        own_text = self._get_element_own_text(element)
-        total_text = own_text + " " + " ".join(
-            child.total_text for child in children
-        )
-        total_text = " ".join(total_text.split())
         
+        if not element.is_displayed():
+            print(f"Skipping invisible {element.tag_name}")
+            return None
+            
+        children = []
+        for child in element.find_elements(By.XPATH, "./*"):
+            if child_node := self._build_tree(child):
+                children.append(child_node)
+                
+        text = self._get_element_text(element)
+        interactions = self._get_interactions(element)
+                
         return DOMNode(
             element=element,
             children=children,
-            own_text=own_text,
+            text=text,
-            total_text=total_text,
             tag=element.tag_name,
             location=element.location,
             size=element.size,
-            is_interactive=self._is_interactive(element),
+            interactions=interactions
-            attributes=self._get_attributes(element)
         )
 
 class ScreenshotGenerator:
-    IGNORED_TAGS = {'script', 'style', 'meta', 'link', 'noscript'}
-    
     def __init__(self, debug_port: int):
+        print(f"\nInitializing ScreenshotGenerator with port {debug_port}")
         options = webdriver.ChromeOptions()
         options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
         self.driver = webdriver.Chrome(options=options)
         
-    def get_viewport_metrics(self) -> Dict[str, int]:
+    def get_viewport_metrics(self) -> Dict[str, str]:
-        return {
+        metrics = {
             "scroll_x": self.driver.execute_script("return window.pageXOffset;"),
             "scroll_y": self.driver.execute_script("return window.pageYOffset;"),
             "width": self.driver.execute_script("return document.documentElement.clientWidth;"),
             "height": self.driver.execute_script("return document.documentElement.clientHeight;"),
             "max_scroll_y": self.driver.execute_script(
                 "return Math.max(document.documentElement.scrollHeight - window.innerHeight, 0);"
+            ),
+            "min_scroll_x": self.driver.execute_script(
+                "return Math.max(document.documentElement.scrollLeft, 0);"
             )
         }
+        print(f"\nViewport metrics: {metrics}")
+        return {k: str(v) for k, v in metrics.items()}
         
-    def should_process_node(self, node: DOMNode) -> bool:
+    def generate_xml(self) -> str:
-        # Skip invisible elements
-        if not node.element.is_displayed():
-            return False
-            
-        # Skip script and style elements
-        if node.tag in self.IGNORED_TAGS:
-            return False
-
-        return True
-        
-    def create_xml_element(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> Optional[ET.Element]:
-        attrs = {
-            "x": str(node.location["x"]),
-            "y": str(node.location["y"]),
-            "width": str(node.size["width"]),
-            "height": str(node.size["height"])
-        }
-        
-        if node.is_interactive:
-            element = ET.Element("interactive", attrs)
-            child = ET.SubElement(element, node.tag)
-            for attr, value in node.attributes.items():
-                child.set(attr, value)
-            if node.own_text:
-                child.text = node.own_text
-            return element
-            
-        element = ET.Element(node.tag, attrs)
-        if node.own_text:
-            element.text = node.own_text
-        return element
-        
-    def process_tree(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> List[ET.Element]:
-        if not self.should_process_node(node):
-            return []
-            
-        current_element = self.create_xml_element(node, parent_element)
-        if not current_element:
-            return []
-            
-        for child in node.children:
-            child_elements = self.process_tree(child, current_element)
-            for child_element in child_elements:
-                current_element.append(child_element)
-                
-        return [current_element]
-        
-    def generate_output(self) -> ET.Element:
         metrics = self.get_viewport_metrics()
-        viewport = ET.Element("viewport", {
+        metrics_str = " ".join(f'{k}="{v}"' for k, v in metrics.items())
-            "scroll_x": str(metrics["scroll_x"]),
-            "scroll_y": str(metrics["scroll_y"]),
-            "width": str(metrics["width"]),
-            "height": str(metrics["height"]),
-            "max_scroll_y": str(metrics["max_scroll_y"])
-        })
         
-        dom_tree = DOMTree(self.driver.find_element(By.TAG_NAME, "body"))
+        output = [f'<viewport {metrics_str}>']
-        for element in self.process_tree(dom_tree.root):
+        seen_texts = set()
-            viewport.append(element)
+        last_was_text = False
-            
+        
-        return viewport
+        body = self.driver.find_element(By.TAG_NAME, "body")
+        dom_tree = DOMTree(body)
+        
+        for content in dom_tree.root.get_content():
+            if isinstance(content, str):
+                if content not in seen_texts:
+                    if last_was_text:
+                        output.append("\n")
+                    output.append(content)
+                    seen_texts.add(content)
+                    last_was_text = True
+            else:
+                output.append(ET.tostring(content, encoding="unicode"))
+                last_was_text = False
+                
+        output.append('</viewport>')
+        return "".join(output)
 
 def main(debug_port: int):
     generator = ScreenshotGenerator(debug_port)
-    output = generator.generate_output()
+    print("\nFinal XML output:")
-    print(ET.tostring(output, encoding="unicode", method="xml"))
+    print(generator.generate_xml())
 
 if __name__ == "__main__":
     if len(sys.argv) != 2:

tools/itb/result.social.html

@@ -1,38 +1,19 @@
-<viewport scroll_x="0" scroll_y="0" width="780" height="441" max_scroll_y="1200">
+<viewport scroll_x="0" scroll_y="0" width="780" height="441" max_scroll_y="1200" max_scroll_x="0">
-    <nav x="0" y="0" width="100%" height="60">
+    SocialConnect
-        <h1>SocialConnect</h1>
+    <interactive x="16" y="0" width="240" height="40" interactions="text_input" placeholder="Search posts..."/>
-        <interactive x="16" y="0" width="240" height="40">
+    Alice Johnson
-            <input placeholder="Search posts..." />
+    2 hours ago
-        </interactive>
+    Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
-    </nav>
+    <img x="36" y="220" width="648" height="400" alt="Sunrise view from mountain top" src="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' width='648' height='400'><rect width='100%' height='100%' fill='%23ddd'/></svg>"/>
-    
+    <interactive x="36" y="632" width="216" height="40" interactions="click">
-    <div class="post" x="20" y="80" width="680" height="400">
+        Like
-        <div class="username" x="72" y="96" width="200" height="20">Alice Johnson</div>
+    </interactive>
-        <div class="timestamp" x="72" y="116" width="200" height="16">2 hours ago</div>
+    <interactive x="252" y="632" width="216" height="40" interactions="click">
-        
+        Comment
-        <div class="post-content" x="36" y="148" width="648" height="60">
+    </interactive>
-            Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
+    <interactive x="468" y="632" width="216" height="40" interactions="click">
-        </div>
+        Share
-        
+    </interactive>
-        <img x="36" y="220" width="648" height="400" alt="Sunrise view from mountain top" />
+    Bob Smith
-        
+    Beautiful view! Which trail is this?
-        <div class="actions" x="36" y="632" width="648" height="40">
-            <interactive x="36" y="632" width="216" height="40">
-                <button>Like</button>
-            </interactive>
-            <interactive x="252" y="632" width="216" height="40">
-                <button>Comment</button>
-            </interactive>
-            <interactive x="468" y="632" width="216" height="40">
-                <button>Share</button>
-            </interactive>
-        </div>
-        
-        <div class="comment" x="44" y="684" width="632" height="60">
-            <div class="username" x="96" y="684" width="200" height="20">Bob Smith</div>
-            <div class="comment-content" x="96" y="704" width="580" height="40">
-                Beautiful view! Which trail is this?
-            </div>
-        </div>
-    </div>
 </viewport>