llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Continue work on itb

Browse Source
This commit is contained in:
geens
2024-12-26 17:36:59 +01:00
parent eadd4eeabe
commit 63d1264f5a
10 changed files with 1281 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
tools/itb/bin/itb_click
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import sys
import os
import time
import random
import argparse
@@ -21,6 +22,13 @@ class ClickExecutor:
options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
self.driver = webdriver.Chrome(options=options)
# Load JavaScript files
js_dir = os.path.join(os.path.dirname(__file__), "..", "js")
self.js_code = ""
for filename in ["viewport.js", "mouse.js"]:
with open(os.path.join(js_dir, filename)) as f:
self.js_code += f.read() + "\n"
# Get initial window info
self.window_size = self.driver.get_window_size()
debug(f"Window size: {json.dumps(self.window_size)}")
@@ -33,36 +41,15 @@ class ClickExecutor:
def get_viewport_size(self):
"""Get viewport dimensions"""
return self.driver.execute_script("""
return {
width: window.innerWidth,
height: window.innerHeight,
pageYOffset: window.pageYOffset,
pageXOffset: window.pageXOffset
};
""")
return self.driver.execute_script(self.js_code + "; return getViewportInfo();")
def get_scroll_position(self):
"""Get current scroll position"""
return self.driver.execute_script("""
return {
scrollX: window.scrollX,
scrollY: window.scrollY,
pageXOffset: window.pageXOffset,
pageYOffset: window.pageYOffset
};
""")
return self.driver.execute_script(self.js_code + "; return getViewportInfo();")
def get_mouse_position(self):
"""Get current mouse coordinates"""
return self.driver.execute_script("""
return {
screenX: window.screenX,
screenY: window.screenY,
clientX: window.event ? window.event.clientX : 0,
clientY: window.event ? window.event.clientY : 0
};
""")
return self.driver.execute_script(self.js_code + "; return getMousePosition();")
def scroll_to_target(self, x: int, y: int):
"""Scroll to make target coordinates visible"""
@@ -78,9 +65,7 @@ class ClickExecutor:
def get_element_at_position(self, x: int, y: int):
"""Get the element at the specified coordinates"""
return self.driver.execute_script("""
return document.elementFromPoint(arguments[0], arguments[1]);
""", x, y)
return self.driver.execute_script(self.js_code + "; return getElementAtPosition(arguments[0], arguments[1]);", x, y)
def execute_click(self, x: int, y: int, double: bool = False, right: bool = False):
"""Execute click operation with improved coordinate handling"""

86
tools/itb/bin/itb_screenshot
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import sys
import os
from selenium import webdriver
from selenium.webdriver.common.by import By
import xml.etree.ElementTree as ET
@@ -10,84 +11,15 @@ class ScreenshotGenerator:
options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
self.driver = webdriver.Chrome(options=options)
# Load JavaScript files
js_dir = os.path.join(os.path.dirname(__file__), "..", "js")
self.js_code = ""
for filename in ["viewport.js", "mouse.js", "dom_analyzer.js"]:
with open(os.path.join(js_dir, filename)) as f:
self.js_code += f.read() + "\n"
def get_page_content(self) -> dict:
script = """
function processElement(element) {
// Check if element is actually visible in viewport
const rect = element.getBoundingClientRect();
if (!element.offsetParent && element !== document.body) return null;
if (rect.width === 0 || rect.height === 0) return null;
// Check if element is within viewport bounds
const viewportWidth = window.innerWidth;
const viewportHeight = window.innerHeight;
if (rect.bottom < 0 || rect.top > viewportHeight ||
rect.right < 0 || rect.left > viewportWidth) {
return null;
}
const styles = window.getComputedStyle(element);
const text = Array.from(element.childNodes)
.filter(node => node.nodeType === Node.TEXT_NODE)
.map(node => node.textContent.trim())
.join(' ');
const interactions = [];
if (element.tagName === 'BUTTON' || element.tagName === 'A' ||
element.getAttribute('role') === 'button' ||
element.getAttribute('onclick') ||
styles.cursor === 'pointer') {
interactions.push('click');
}
if (element.tagName === 'INPUT' || element.tagName === 'TEXTAREA' ||
element.getAttribute('contenteditable') === 'true') {
interactions.push('text_input');
const placeholder = element.getAttribute('placeholder');
if (placeholder) interactions.push('placeholder:' + placeholder);
}
if (element.tagName === 'SELECT' ||
element.getAttribute('role') === 'listbox' ||
element.getAttribute('type') === 'checkbox' ||
element.getAttribute('type') === 'radio') {
interactions.push('select');
}
if (element.scrollHeight > element.clientHeight ||
element.scrollWidth > element.clientWidth) {
interactions.push('scroll');
}
const children = Array.from(element.children)
.map(child => processElement(child))
.filter(child => child !== null);
return {
tag: element.tagName.toLowerCase(),
text,
location: {x: Math.round(rect.left), y: Math.round(rect.top)},
size: {width: Math.round(rect.width), height: Math.round(rect.height)},
interactions,
children
};
}
const body = document.body;
return {
viewport: {
scroll_x: window.pageXOffset,
scroll_y: window.pageYOffset,
width: document.documentElement.clientWidth,
height: document.documentElement.clientHeight,
max_scroll_y: Math.max(document.documentElement.scrollHeight - window.innerHeight, 0),
min_scroll_x: Math.max(document.documentElement.scrollLeft, 0)
},
content: body ? processElement(body) : null
};
"""
return self.driver.execute_script(script)
return self.driver.execute_script(self.js_code + "; return analyzePageContent();")
def format_xml(self, xml_str: str) -> str:
root = ET.fromstring(xml_str)

17
tools/itb/bin/itb_scroll
View File

@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import sys
import os
import time
import random
from selenium import webdriver
@@ -9,17 +10,15 @@ class ScrollController:
options = webdriver.ChromeOptions()
options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
self.driver = webdriver.Chrome(options=options)
# Load JavaScript files
js_dir = os.path.join(os.path.dirname(__file__), "..", "js")
self.js_code = ""
with open(os.path.join(js_dir, "viewport.js")) as f:
self.js_code = f.read()
def get_viewport_info(self) -> dict:
script = """
return {
scroll_x: window.pageXOffset,
scroll_y: window.pageYOffset,
max_scroll_x: Math.max(document.documentElement.scrollWidth - window.innerWidth, 0),
max_scroll_y: Math.max(document.documentElement.scrollHeight - window.innerHeight, 0)
};
"""
return self.driver.execute_script(script)
return self.driver.execute_script(self.js_code + "; return getViewportInfo();")
def smooth_scroll(self, target_x: int, target_y: int):
viewport = self.get_viewport_info()

2
tools/itb/bin/itb_start
View File

@@ -7,7 +7,7 @@ def start(url):
debug_port = random.randint(9000, 9999)
chrome_cmd = [
'/usr/bin/google-chrome',
'--headless=new',
#'--headless=new',
'--no-sandbox',
'--remote-debugging-port=' + str(debug_port),
url