ITB WIP
This commit is contained in:
Niels Geens
@@ -4,174 +4,194 @@ from selenium import webdriver
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.remote.webelement import WebElement
|
||||
import xml.etree.ElementTree as ET
|
||||
from typing import Dict, List, Set, Optional
|
||||
from typing import Dict, List, Set, Optional, Union
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class DOMNode:
|
||||
element: WebElement
|
||||
children: List['DOMNode']
|
||||
own_text: str
|
||||
total_text: str
|
||||
text: str
|
||||
tag: str
|
||||
location: Dict[str, int]
|
||||
size: Dict[str, int]
|
||||
is_interactive: bool
|
||||
attributes: Dict[str, str]
|
||||
interactions: List[str]
|
||||
|
||||
def get_content(self) -> List[Union[ET.Element, str]]:
|
||||
content = []
|
||||
|
||||
# Add text as plain string if present
|
||||
if self.text:
|
||||
content.append(self.text)
|
||||
|
||||
# Add interactive element if present
|
||||
if self.interactions:
|
||||
attrs = {
|
||||
"x": str(self.location["x"]),
|
||||
"y": str(self.location["y"]),
|
||||
"width": str(self.size["width"]),
|
||||
"height": str(self.size["height"])
|
||||
}
|
||||
|
||||
# Split interactions and placeholders
|
||||
base_interactions = []
|
||||
placeholder = None
|
||||
|
||||
for interaction in self.interactions:
|
||||
if interaction.startswith("placeholder:"):
|
||||
placeholder = interaction.split(":", 1)[1]
|
||||
else:
|
||||
base_interactions.append(interaction)
|
||||
|
||||
attrs["interactions"] = ",".join(base_interactions)
|
||||
if placeholder:
|
||||
attrs["placeholder"] = placeholder
|
||||
|
||||
interactive = ET.Element("interactive", attrs)
|
||||
if self.text:
|
||||
interactive.text = self.text
|
||||
content.append(interactive)
|
||||
|
||||
# Add all children's content
|
||||
for child in self.children:
|
||||
content.extend(child.get_content())
|
||||
|
||||
return content
|
||||
|
||||
class DOMTree:
|
||||
def __init__(self, root_element: WebElement):
|
||||
print(f"\nBuilding DOM tree for {root_element.tag_name}")
|
||||
self.root = self._build_tree(root_element)
|
||||
|
||||
def _get_element_own_text(self, element: WebElement) -> str:
|
||||
def _get_element_text(self, element: WebElement) -> str:
|
||||
script = """
|
||||
let element = arguments[0];
|
||||
let walker = document.createTreeWalker(
|
||||
element,
|
||||
NodeFilter.SHOW_TEXT,
|
||||
{
|
||||
acceptNode: function(node) {
|
||||
return node.parentElement === element ?
|
||||
NodeFilter.FILTER_ACCEPT :
|
||||
NodeFilter.FILTER_REJECT;
|
||||
}
|
||||
}
|
||||
);
|
||||
let text = '';
|
||||
let node;
|
||||
while (node = walker.nextNode()) {
|
||||
text += node.textContent;
|
||||
for (let node of element.childNodes) {
|
||||
if (node.nodeType === Node.TEXT_NODE) {
|
||||
text += node.textContent;
|
||||
}
|
||||
}
|
||||
return text.trim();
|
||||
"""
|
||||
return element.parent.execute_script(script, element)
|
||||
text = element.parent.execute_script(script, element)
|
||||
if text:
|
||||
print(f"Got text: {text[:50]}...")
|
||||
return text
|
||||
|
||||
def _is_interactive(self, element: WebElement) -> bool:
|
||||
return (
|
||||
element.tag_name in {"button", "a", "input", "select", "textarea"} or
|
||||
element.get_attribute("role") in {"button", "link", "textbox"} or
|
||||
element.get_attribute("onclick") is not None or
|
||||
element.get_attribute("contenteditable") == "true"
|
||||
)
|
||||
def _get_interactions(self, element: WebElement) -> List[str]:
|
||||
interactions = []
|
||||
|
||||
def _get_attributes(self, element: WebElement) -> Dict[str, str]:
|
||||
attrs = {}
|
||||
for attr in ['placeholder', 'aria-label', 'alt', 'title', 'role']:
|
||||
if value := element.get_attribute(attr):
|
||||
attrs[attr] = value
|
||||
return attrs
|
||||
if (element.tag_name in {"button", "a"} or
|
||||
element.get_attribute("role") in {"button", "link"} or
|
||||
element.get_attribute("onclick") or
|
||||
element.value_of_css_property("cursor") == "pointer"):
|
||||
interactions.append("click")
|
||||
|
||||
if (element.tag_name in {"input", "textarea"} or
|
||||
element.get_attribute("contenteditable") == "true" or
|
||||
element.get_attribute("role") == "textbox"):
|
||||
interactions.append("text_input")
|
||||
# Get placeholder if available
|
||||
placeholder = element.get_attribute("placeholder")
|
||||
if placeholder:
|
||||
interactions.append(f"placeholder:{placeholder}")
|
||||
|
||||
if (element.tag_name == "select" or
|
||||
element.get_attribute("role") == "listbox" or
|
||||
element.get_attribute("type") in {"checkbox", "radio"}):
|
||||
interactions.append("select")
|
||||
|
||||
script = """
|
||||
let el = arguments[0];
|
||||
return el.scrollHeight > el.clientHeight ||
|
||||
el.scrollWidth > el.clientWidth;
|
||||
"""
|
||||
if element.parent.execute_script(script, element):
|
||||
interactions.append("scroll")
|
||||
|
||||
if interactions:
|
||||
print(f"Found interactions for {element.tag_name}: {interactions}")
|
||||
return interactions
|
||||
|
||||
def _build_tree(self, element: WebElement) -> DOMNode:
|
||||
children = [
|
||||
self._build_tree(child)
|
||||
for child in element.find_elements(By.XPATH, "./*")
|
||||
]
|
||||
|
||||
own_text = self._get_element_own_text(element)
|
||||
total_text = own_text + " " + " ".join(
|
||||
child.total_text for child in children
|
||||
)
|
||||
total_text = " ".join(total_text.split())
|
||||
print(f"\nProcessing <{element.tag_name}>")
|
||||
|
||||
if not element.is_displayed():
|
||||
print(f"Skipping invisible {element.tag_name}")
|
||||
return None
|
||||
|
||||
children = []
|
||||
for child in element.find_elements(By.XPATH, "./*"):
|
||||
if child_node := self._build_tree(child):
|
||||
children.append(child_node)
|
||||
|
||||
text = self._get_element_text(element)
|
||||
interactions = self._get_interactions(element)
|
||||
|
||||
return DOMNode(
|
||||
element=element,
|
||||
children=children,
|
||||
own_text=own_text,
|
||||
total_text=total_text,
|
||||
text=text,
|
||||
tag=element.tag_name,
|
||||
location=element.location,
|
||||
size=element.size,
|
||||
is_interactive=self._is_interactive(element),
|
||||
attributes=self._get_attributes(element)
|
||||
interactions=interactions
|
||||
)
|
||||
|
||||
class ScreenshotGenerator:
|
||||
IGNORED_TAGS = {'script', 'style', 'meta', 'link', 'noscript'}
|
||||
|
||||
def __init__(self, debug_port: int):
|
||||
print(f"\nInitializing ScreenshotGenerator with port {debug_port}")
|
||||
options = webdriver.ChromeOptions()
|
||||
options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
|
||||
self.driver = webdriver.Chrome(options=options)
|
||||
|
||||
def get_viewport_metrics(self) -> Dict[str, int]:
|
||||
return {
|
||||
def get_viewport_metrics(self) -> Dict[str, str]:
|
||||
metrics = {
|
||||
"scroll_x": self.driver.execute_script("return window.pageXOffset;"),
|
||||
"scroll_y": self.driver.execute_script("return window.pageYOffset;"),
|
||||
"width": self.driver.execute_script("return document.documentElement.clientWidth;"),
|
||||
"height": self.driver.execute_script("return document.documentElement.clientHeight;"),
|
||||
"max_scroll_y": self.driver.execute_script(
|
||||
"return Math.max(document.documentElement.scrollHeight - window.innerHeight, 0);"
|
||||
),
|
||||
"min_scroll_x": self.driver.execute_script(
|
||||
"return Math.max(document.documentElement.scrollLeft, 0);"
|
||||
)
|
||||
}
|
||||
print(f"\nViewport metrics: {metrics}")
|
||||
return {k: str(v) for k, v in metrics.items()}
|
||||
|
||||
def should_process_node(self, node: DOMNode) -> bool:
|
||||
# Skip invisible elements
|
||||
if not node.element.is_displayed():
|
||||
return False
|
||||
|
||||
# Skip script and style elements
|
||||
if node.tag in self.IGNORED_TAGS:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def create_xml_element(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> Optional[ET.Element]:
|
||||
attrs = {
|
||||
"x": str(node.location["x"]),
|
||||
"y": str(node.location["y"]),
|
||||
"width": str(node.size["width"]),
|
||||
"height": str(node.size["height"])
|
||||
}
|
||||
|
||||
if node.is_interactive:
|
||||
element = ET.Element("interactive", attrs)
|
||||
child = ET.SubElement(element, node.tag)
|
||||
for attr, value in node.attributes.items():
|
||||
child.set(attr, value)
|
||||
if node.own_text:
|
||||
child.text = node.own_text
|
||||
return element
|
||||
|
||||
element = ET.Element(node.tag, attrs)
|
||||
if node.own_text:
|
||||
element.text = node.own_text
|
||||
return element
|
||||
|
||||
def process_tree(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> List[ET.Element]:
|
||||
if not self.should_process_node(node):
|
||||
return []
|
||||
|
||||
current_element = self.create_xml_element(node, parent_element)
|
||||
if not current_element:
|
||||
return []
|
||||
|
||||
for child in node.children:
|
||||
child_elements = self.process_tree(child, current_element)
|
||||
for child_element in child_elements:
|
||||
current_element.append(child_element)
|
||||
|
||||
return [current_element]
|
||||
|
||||
def generate_output(self) -> ET.Element:
|
||||
def generate_xml(self) -> str:
|
||||
metrics = self.get_viewport_metrics()
|
||||
viewport = ET.Element("viewport", {
|
||||
"scroll_x": str(metrics["scroll_x"]),
|
||||
"scroll_y": str(metrics["scroll_y"]),
|
||||
"width": str(metrics["width"]),
|
||||
"height": str(metrics["height"]),
|
||||
"max_scroll_y": str(metrics["max_scroll_y"])
|
||||
})
|
||||
metrics_str = " ".join(f'{k}="{v}"' for k, v in metrics.items())
|
||||
|
||||
dom_tree = DOMTree(self.driver.find_element(By.TAG_NAME, "body"))
|
||||
for element in self.process_tree(dom_tree.root):
|
||||
viewport.append(element)
|
||||
|
||||
return viewport
|
||||
output = [f'<viewport {metrics_str}>']
|
||||
seen_texts = set()
|
||||
last_was_text = False
|
||||
|
||||
body = self.driver.find_element(By.TAG_NAME, "body")
|
||||
dom_tree = DOMTree(body)
|
||||
|
||||
for content in dom_tree.root.get_content():
|
||||
if isinstance(content, str):
|
||||
if content not in seen_texts:
|
||||
if last_was_text:
|
||||
output.append("\n")
|
||||
output.append(content)
|
||||
seen_texts.add(content)
|
||||
last_was_text = True
|
||||
else:
|
||||
output.append(ET.tostring(content, encoding="unicode"))
|
||||
last_was_text = False
|
||||
|
||||
output.append('</viewport>')
|
||||
return "".join(output)
|
||||
|
||||
def main(debug_port: int):
|
||||
generator = ScreenshotGenerator(debug_port)
|
||||
output = generator.generate_output()
|
||||
print(ET.tostring(output, encoding="unicode", method="xml"))
|
||||
print("\nFinal XML output:")
|
||||
print(generator.generate_xml())
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 2:
|
||||
|
||||
@@ -1,38 +1,19 @@
|
||||
<viewport scroll_x="0" scroll_y="0" width="780" height="441" max_scroll_y="1200">
|
||||
<nav x="0" y="0" width="100%" height="60">
|
||||
<h1>SocialConnect</h1>
|
||||
<interactive x="16" y="0" width="240" height="40">
|
||||
<input placeholder="Search posts..." />
|
||||
</interactive>
|
||||
</nav>
|
||||
|
||||
<div class="post" x="20" y="80" width="680" height="400">
|
||||
<div class="username" x="72" y="96" width="200" height="20">Alice Johnson</div>
|
||||
<div class="timestamp" x="72" y="116" width="200" height="16">2 hours ago</div>
|
||||
|
||||
<div class="post-content" x="36" y="148" width="648" height="60">
|
||||
Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
|
||||
</div>
|
||||
|
||||
<img x="36" y="220" width="648" height="400" alt="Sunrise view from mountain top" />
|
||||
|
||||
<div class="actions" x="36" y="632" width="648" height="40">
|
||||
<interactive x="36" y="632" width="216" height="40">
|
||||
<button>Like</button>
|
||||
</interactive>
|
||||
<interactive x="252" y="632" width="216" height="40">
|
||||
<button>Comment</button>
|
||||
</interactive>
|
||||
<interactive x="468" y="632" width="216" height="40">
|
||||
<button>Share</button>
|
||||
</interactive>
|
||||
</div>
|
||||
|
||||
<div class="comment" x="44" y="684" width="632" height="60">
|
||||
<div class="username" x="96" y="684" width="200" height="20">Bob Smith</div>
|
||||
<div class="comment-content" x="96" y="704" width="580" height="40">
|
||||
Beautiful view! Which trail is this?
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<viewport scroll_x="0" scroll_y="0" width="780" height="441" max_scroll_y="1200" max_scroll_x="0">
|
||||
SocialConnect
|
||||
<interactive x="16" y="0" width="240" height="40" interactions="text_input" placeholder="Search posts..."/>
|
||||
Alice Johnson
|
||||
2 hours ago
|
||||
Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
|
||||
<img x="36" y="220" width="648" height="400" alt="Sunrise view from mountain top" src="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' width='648' height='400'><rect width='100%' height='100%' fill='%23ddd'/></svg>"/>
|
||||
<interactive x="36" y="632" width="216" height="40" interactions="click">
|
||||
Like
|
||||
</interactive>
|
||||
<interactive x="252" y="632" width="216" height="40" interactions="click">
|
||||
Comment
|
||||
</interactive>
|
||||
<interactive x="468" y="632" width="216" height="40" interactions="click">
|
||||
Share
|
||||
</interactive>
|
||||
Bob Smith
|
||||
Beautiful view! Which trail is this?
|
||||
</viewport>
|
||||
|
||||
Reference in New Issue
Block a user