diff --git a/container.sh b/container.sh
index 273376d..741619f 100755
--- a/container.sh
+++ b/container.sh
@@ -1,4 +1,4 @@
#!/bin/bash
container_id=$(docker ps -q)
-docker exec -it $container_id /bin/bash
\ No newline at end of file
+docker exec -it $container_id bash
\ No newline at end of file
diff --git a/run.sh b/run.sh
index aa09932..886d915 100755
--- a/run.sh
+++ b/run.sh
@@ -25,5 +25,6 @@ docker run \
-v /$(pwd)/model/:/root/model/ \
-v /$(pwd)/.env:/root/.env \
-v /$(pwd)/iterations/:/root/iterations/ \
+ -v /$(pwd)/tools/:/root/sia/tools/ \
-v /$(pwd)/.git/:/root/sia/.git/ \
sia "$@"
diff --git a/tools/itb/bin/itb_screenshot b/tools/itb/bin/itb_screenshot
index 44d8bf6..5440e10 100755
--- a/tools/itb/bin/itb_screenshot
+++ b/tools/itb/bin/itb_screenshot
@@ -4,174 +4,194 @@ from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.remote.webelement import WebElement
import xml.etree.ElementTree as ET
-from typing import Dict, List, Set, Optional
+from typing import Dict, List, Set, Optional, Union
from dataclasses import dataclass
@dataclass
class DOMNode:
element: WebElement
children: List['DOMNode']
- own_text: str
- total_text: str
+ text: str
tag: str
location: Dict[str, int]
size: Dict[str, int]
- is_interactive: bool
- attributes: Dict[str, str]
+ interactions: List[str]
+
+ def get_content(self) -> List[Union[ET.Element, str]]:
+ content = []
+
+ # Add text as plain string if present
+ if self.text:
+ content.append(self.text)
+
+ # Add interactive element if present
+ if self.interactions:
+ attrs = {
+ "x": str(self.location["x"]),
+ "y": str(self.location["y"]),
+ "width": str(self.size["width"]),
+ "height": str(self.size["height"])
+ }
+
+ # Split interactions and placeholders
+ base_interactions = []
+ placeholder = None
+
+ for interaction in self.interactions:
+ if interaction.startswith("placeholder:"):
+ placeholder = interaction.split(":", 1)[1]
+ else:
+ base_interactions.append(interaction)
+
+ attrs["interactions"] = ",".join(base_interactions)
+ if placeholder:
+ attrs["placeholder"] = placeholder
+
+ interactive = ET.Element("interactive", attrs)
+ if self.text:
+ interactive.text = self.text
+ content.append(interactive)
+
+ # Add all children's content
+ for child in self.children:
+ content.extend(child.get_content())
+
+ return content
class DOMTree:
def __init__(self, root_element: WebElement):
+ print(f"\nBuilding DOM tree for {root_element.tag_name}")
self.root = self._build_tree(root_element)
- def _get_element_own_text(self, element: WebElement) -> str:
+ def _get_element_text(self, element: WebElement) -> str:
script = """
let element = arguments[0];
- let walker = document.createTreeWalker(
- element,
- NodeFilter.SHOW_TEXT,
- {
- acceptNode: function(node) {
- return node.parentElement === element ?
- NodeFilter.FILTER_ACCEPT :
- NodeFilter.FILTER_REJECT;
- }
- }
- );
let text = '';
- let node;
- while (node = walker.nextNode()) {
- text += node.textContent;
+ for (let node of element.childNodes) {
+ if (node.nodeType === Node.TEXT_NODE) {
+ text += node.textContent;
+ }
}
return text.trim();
"""
- return element.parent.execute_script(script, element)
+ text = element.parent.execute_script(script, element)
+ if text:
+ print(f"Got text: {text[:50]}...")
+ return text
- def _is_interactive(self, element: WebElement) -> bool:
- return (
- element.tag_name in {"button", "a", "input", "select", "textarea"} or
- element.get_attribute("role") in {"button", "link", "textbox"} or
- element.get_attribute("onclick") is not None or
- element.get_attribute("contenteditable") == "true"
- )
+ def _get_interactions(self, element: WebElement) -> List[str]:
+ interactions = []
- def _get_attributes(self, element: WebElement) -> Dict[str, str]:
- attrs = {}
- for attr in ['placeholder', 'aria-label', 'alt', 'title', 'role']:
- if value := element.get_attribute(attr):
- attrs[attr] = value
- return attrs
+ if (element.tag_name in {"button", "a"} or
+ element.get_attribute("role") in {"button", "link"} or
+ element.get_attribute("onclick") or
+ element.value_of_css_property("cursor") == "pointer"):
+ interactions.append("click")
+
+ if (element.tag_name in {"input", "textarea"} or
+ element.get_attribute("contenteditable") == "true" or
+ element.get_attribute("role") == "textbox"):
+ interactions.append("text_input")
+ # Get placeholder if available
+ placeholder = element.get_attribute("placeholder")
+ if placeholder:
+ interactions.append(f"placeholder:{placeholder}")
+
+ if (element.tag_name == "select" or
+ element.get_attribute("role") == "listbox" or
+ element.get_attribute("type") in {"checkbox", "radio"}):
+ interactions.append("select")
+
+ script = """
+ let el = arguments[0];
+ return el.scrollHeight > el.clientHeight ||
+ el.scrollWidth > el.clientWidth;
+ """
+ if element.parent.execute_script(script, element):
+ interactions.append("scroll")
+
+ if interactions:
+ print(f"Found interactions for {element.tag_name}: {interactions}")
+ return interactions
def _build_tree(self, element: WebElement) -> DOMNode:
- children = [
- self._build_tree(child)
- for child in element.find_elements(By.XPATH, "./*")
- ]
-
- own_text = self._get_element_own_text(element)
- total_text = own_text + " " + " ".join(
- child.total_text for child in children
- )
- total_text = " ".join(total_text.split())
+ print(f"\nProcessing <{element.tag_name}>")
+ if not element.is_displayed():
+ print(f"Skipping invisible {element.tag_name}")
+ return None
+
+ children = []
+ for child in element.find_elements(By.XPATH, "./*"):
+ if child_node := self._build_tree(child):
+ children.append(child_node)
+
+ text = self._get_element_text(element)
+ interactions = self._get_interactions(element)
+
return DOMNode(
element=element,
children=children,
- own_text=own_text,
- total_text=total_text,
+ text=text,
tag=element.tag_name,
location=element.location,
size=element.size,
- is_interactive=self._is_interactive(element),
- attributes=self._get_attributes(element)
+ interactions=interactions
)
class ScreenshotGenerator:
- IGNORED_TAGS = {'script', 'style', 'meta', 'link', 'noscript'}
-
def __init__(self, debug_port: int):
+ print(f"\nInitializing ScreenshotGenerator with port {debug_port}")
options = webdriver.ChromeOptions()
options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
self.driver = webdriver.Chrome(options=options)
- def get_viewport_metrics(self) -> Dict[str, int]:
- return {
+ def get_viewport_metrics(self) -> Dict[str, str]:
+ metrics = {
"scroll_x": self.driver.execute_script("return window.pageXOffset;"),
"scroll_y": self.driver.execute_script("return window.pageYOffset;"),
"width": self.driver.execute_script("return document.documentElement.clientWidth;"),
"height": self.driver.execute_script("return document.documentElement.clientHeight;"),
"max_scroll_y": self.driver.execute_script(
"return Math.max(document.documentElement.scrollHeight - window.innerHeight, 0);"
+ ),
+ "min_scroll_x": self.driver.execute_script(
+ "return Math.max(document.documentElement.scrollLeft, 0);"
)
}
+ print(f"\nViewport metrics: {metrics}")
+ return {k: str(v) for k, v in metrics.items()}
- def should_process_node(self, node: DOMNode) -> bool:
- # Skip invisible elements
- if not node.element.is_displayed():
- return False
-
- # Skip script and style elements
- if node.tag in self.IGNORED_TAGS:
- return False
-
- return True
-
- def create_xml_element(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> Optional[ET.Element]:
- attrs = {
- "x": str(node.location["x"]),
- "y": str(node.location["y"]),
- "width": str(node.size["width"]),
- "height": str(node.size["height"])
- }
-
- if node.is_interactive:
- element = ET.Element("interactive", attrs)
- child = ET.SubElement(element, node.tag)
- for attr, value in node.attributes.items():
- child.set(attr, value)
- if node.own_text:
- child.text = node.own_text
- return element
-
- element = ET.Element(node.tag, attrs)
- if node.own_text:
- element.text = node.own_text
- return element
-
- def process_tree(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> List[ET.Element]:
- if not self.should_process_node(node):
- return []
-
- current_element = self.create_xml_element(node, parent_element)
- if not current_element:
- return []
-
- for child in node.children:
- child_elements = self.process_tree(child, current_element)
- for child_element in child_elements:
- current_element.append(child_element)
-
- return [current_element]
-
- def generate_output(self) -> ET.Element:
+ def generate_xml(self) -> str:
metrics = self.get_viewport_metrics()
- viewport = ET.Element("viewport", {
- "scroll_x": str(metrics["scroll_x"]),
- "scroll_y": str(metrics["scroll_y"]),
- "width": str(metrics["width"]),
- "height": str(metrics["height"]),
- "max_scroll_y": str(metrics["max_scroll_y"])
- })
+ metrics_str = " ".join(f'{k}="{v}"' for k, v in metrics.items())
- dom_tree = DOMTree(self.driver.find_element(By.TAG_NAME, "body"))
- for element in self.process_tree(dom_tree.root):
- viewport.append(element)
-
- return viewport
+ output = [f'']
+ seen_texts = set()
+ last_was_text = False
+
+ body = self.driver.find_element(By.TAG_NAME, "body")
+ dom_tree = DOMTree(body)
+
+ for content in dom_tree.root.get_content():
+ if isinstance(content, str):
+ if content not in seen_texts:
+ if last_was_text:
+ output.append("\n")
+ output.append(content)
+ seen_texts.add(content)
+ last_was_text = True
+ else:
+ output.append(ET.tostring(content, encoding="unicode"))
+ last_was_text = False
+
+ output.append('')
+ return "".join(output)
def main(debug_port: int):
generator = ScreenshotGenerator(debug_port)
- output = generator.generate_output()
- print(ET.tostring(output, encoding="unicode", method="xml"))
+ print("\nFinal XML output:")
+ print(generator.generate_xml())
if __name__ == "__main__":
if len(sys.argv) != 2:
diff --git a/tools/itb/result.social.html b/tools/itb/result.social.html
index e1bdf4b..c87abf5 100644
--- a/tools/itb/result.social.html
+++ b/tools/itb/result.social.html
@@ -1,38 +1,19 @@
-
-
-
-
-
Alice Johnson
-
2 hours ago
-
-
- Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Bob Smith
-
- Beautiful view! Which trail is this?
-
-
-
+
+ SocialConnect
+
+ Alice Johnson
+ 2 hours ago
+ Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
+
+
+ Like
+
+
+ Comment
+
+
+ Share
+
+ Bob Smith
+ Beautiful view! Which trail is this?