ITP WiP

Dockerfile

@@ -28,6 +28,20 @@ COPY web .
 RUN npm run build
 
 FROM requirements
+RUN apt-get update
+RUN apt-get install -y wget gnupg
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
+RUN echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list
+RUN apt-get update
+RUN apt-get install -y google-chrome-stable
+RUN rm -rf /var/lib/apt/lists/*
+
+COPY ./tools/itb/requirements.txt /root/sia/tools/itb/requirements.txt
+RUN cd /root/sia/tools/itb/ && python3 -m pip install -r requirements.txt
+
+COPY ./tools/ /root/sia/tools/
+RUN cd /root/sia/tools/itb/ && python3 -m pip install -e ".[dev]"
+
 COPY ./ /root/sia/
 COPY --from=web-build /app/dist /root/sia/static/
 WORKDIR /root/sia

container.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+container_id=$(docker ps -q)
+docker exec -it $container_id /bin/bash

selenium/dom.py

@@ -1,38 +0,0 @@
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-import json
-
-def connect_to_chrome():
-    """
-    Connect to an existing Chrome browser instance and print the DOM
-    """
-    # Chrome options for connecting to existing browser
-    chrome_options = Options()
-    chrome_options.add_experimental_option("debuggerAddress", "127.0.0.1:1234")
-    
-    try:
-        # Connect to the existing browser
-        driver = webdriver.Chrome(options=chrome_options)
-        
-        # Get the current page's DOM
-        dom = driver.find_element(By.TAG_NAME, 'html').get_attribute('outerHTML')
-        
-        # Pretty print the DOM
-        parsed_dom = json.loads(json.dumps(dom))
-        print(json.dumps(parsed_dom, indent=2))
-        
-        return driver
-        
-    except Exception as e:
-        print(f"Error connecting to Chrome: {str(e)}")
-        print("\nMake sure Chrome is running with remote debugging enabled!")
-        print("Start Chrome with: chrome.exe --remote-debugging-port=9222")
-        return None
-
-if __name__ == "__main__":
-    driver = connect_to_chrome()
-    if driver:
-        # Keep the connection open until user input
-        input("Press Enter to close the connection...")
-        driver.quit()

selenium/install.sh

@@ -1,6 +0,0 @@
-apt-get update && apt-get install gnupg wget -y && \
-    wget --quiet --output-document=- https://dl-ssl.google.com/linux/linux_signing_key.pub | gpg --dearmor > /etc/apt/trusted.gpg.d/google-archive.gpg && \
-    sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' && \
-    apt-get update && \
-    apt-get install google-chrome-stable -y --no-install-recommends && \
-    rm -rf /var/lib/apt/lists/*

selenium/start.sh

@@ -1,5 +0,0 @@
-/opt/google/chrome/chrome \
---no-sandbox \
---headless=new \
---remote-debugging-port=1234 \
-https://google.be &

sia/__main__.py

@@ -73,13 +73,13 @@ class Main:
             metrics=SystemMetrics(),
             llms=self._llms,
             validator=XMLValidator(self._action_schema),
-            parser=ResponseParser(self._io_buffer),
+            parser=ResponseParser(config.work_dir, self._io_buffer),
             iteration_logger=IterationLogger(self._config.iterations_dir, self._system_prompt, self._action_schema),
         )
         self._auto_approver = AutoApprover(self._agent)
 
         self._app = web.Application()
-        self._api = Api(self._app, self._agent, self._io_buffer, self._working_memory, self._auto_approver)
+        self._api = Api(config.work_dir, self._app, self._agent, self._io_buffer, self._working_memory, self._auto_approver)
         self._websockets = Websockets(self._app, self._agent, self._io_buffer, self._auto_approver, self._working_memory)
         self._static = Static(self._app, self._config)
 

sia/config.py

@@ -30,6 +30,12 @@ class Config:
             default=os.getenv('SIA_ITERATIONS_DIR', 'iterations'),
             help='Path to the directory for storing iterations (default: iterations, env: SIA_ITERATIONS_DIR)'
         )
+        parser.add_argument(
+            '--work-dir',
+            type=Path,
+            default=os.getenv('SIA_WORK_DIR', '/root'),
+            help='Path to the working directory (default: /root, env: SIA_WORK_DIR)'
+        )
 
         # Web server configuration
         parser.add_argument(
@@ -206,6 +212,10 @@ class Config:
     def iterations_dir(self) -> Path:
         return self.args.iterations_dir
     
+    @property
+    def work_dir(self) -> Path:
+        return self.args.work_dir
+
     # Server properties
     @property
     def server(self) -> bool:

sia/entry/background_entry.py

@@ -19,6 +19,7 @@ class BackgroundEntry(Entry):
     def __init__(
             self,
             id: str,
+            work_dir: str,
             script: str,
         ):
         """
@@ -26,9 +27,11 @@ class BackgroundEntry(Entry):
         
         Args:
             id: Unique identifier for this entry
+            work_dir: Working directory for the process
             script: The script/command to execute
         """
         super().__init__(id)
+        self.work_dir = work_dir
         self.script = script
         self._process: Optional[subprocess.Popen] = None
         self.stdout = ""
@@ -81,6 +84,7 @@ class BackgroundEntry(Entry):
         if self._process is None and self.exit_code is None:
             self._process = subprocess.Popen(
                 self.script,
+                cwd=self.work_dir,
                 shell=True,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.PIPE,
@@ -155,6 +159,7 @@ class BackgroundEntry(Entry):
         return {
             "type": "background",
             "id": self.id,
+            "work_dir": str(self.work_dir),
             "script": self.script,
             "stdout": self.stdout,
             "stderr": self.stderr,

sia/entry/entry_factory.py

@@ -1,4 +1,4 @@
-from typing import Optional
+from pathlib import Path
 
 from . import Entry
 from ..io_buffer import IOBuffer
@@ -11,12 +11,12 @@ from .single_entry import SingleEntry
 from .write_entry import WriteEntry
 
 class EntryFactory:
-    def create_entry(data: dict, io_buffer: IOBuffer) -> Entry:
+    def create_entry(data: dict, work_dir: Path, io_buffer: IOBuffer) -> Entry:
         entry_type = data.get("type")
         entry_id = data.get("id")
         
         if entry_type == "background":
-            return BackgroundEntry(entry_id, data["script"])
+            return BackgroundEntry(entry_id, work_dir, data["script"])
             
         elif entry_type == "read_stdin":
             return ReadEntry(entry_id, io_buffer)
@@ -27,6 +27,7 @@ class EntryFactory:
         elif entry_type == "repeat":
             return RepeatEntry(
                 entry_id,
+                work_dir,
                 data["script"],
                 data.get("timeout"),
                 data.get("limit")
@@ -35,6 +36,7 @@ class EntryFactory:
         elif entry_type == "single":
             return SingleEntry(
                 entry_id,
+                work_dir,
                 data["script"],
                 data.get("timeout"),
                 data.get("limit")
@@ -51,6 +53,8 @@ class EntryFactory:
                 entry.id = data["id"]
 
         if isinstance(entry, SingleEntry):
+            if "work_dir" in data:
+                entry.work_dir = Path(data["work_dir"])
             if "script" in data:
                 entry.script = data["script"]
             if "timeout" in data:
@@ -69,6 +73,8 @@ class EntryFactory:
                 entry.timed_out = bool(data["timed_out"])
 
         if isinstance(entry, RepeatEntry):
+            if "work_dir" in data:
+                entry.work_dir = Path(data["work_dir"])
             if "script" in data:
                 entry.script = data["script"]
             if "timeout" in data:
@@ -87,6 +93,8 @@ class EntryFactory:
                 entry.timed_out = bool(data["timed_out"])
 
         elif isinstance(entry, BackgroundEntry):
+            if "work_dir" in data:
+                entry.work_dir = Path(data["work_dir"])
             if "script" in data:
                 entry.script = data["script"]
             if "stdout" in data:

sia/entry/repeat_entry.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import subprocess
 import xml.etree.ElementTree as ET
 from typing import Optional
@@ -15,6 +16,7 @@ class RepeatEntry(Entry):
     def __init__(
             self,
             id: str,
+            work_dir: Path,
             script: str,
             timeout: Optional[float] = None,
             limit: Optional[int] = None,
@@ -29,6 +31,7 @@ class RepeatEntry(Entry):
             limit: Maximum number of characters to capture from stdout/stderr
         """
         super().__init__(id)
+        self.work_dir = work_dir
         self.script = script
         self.timeout = timeout
         self.limit = limit
@@ -45,6 +48,7 @@ class RepeatEntry(Entry):
         try:
             process = subprocess.run(
                 self.script,
+                cwd=self.work_dir,
                 timeout=(self.timeout or self.default_timeout),
                 shell=True,
                 capture_output=True,
@@ -95,6 +99,7 @@ class RepeatEntry(Entry):
         return {
             "type": "repeat",
             "id": self.id,
+            "work_dir": str(self.work_dir),
             "script": self.script,
             "timeout": self.timeout,
             "limit": self.limit,

sia/entry/single_entry.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 import subprocess
 import xml.etree.ElementTree as ET
 from typing import Optional
@@ -15,6 +16,7 @@ class SingleEntry(Entry):
     def __init__(
             self,
             id: str,
+            work_dir: Path,
             script: str,
             timeout: Optional[float] = None,
             limit: Optional[int] = None,
@@ -29,6 +31,7 @@ class SingleEntry(Entry):
             limit: Maximum number of characters to capture from stdout/stderr
         """
         super().__init__(id)
+        self.work_dir = work_dir
         self.script = script
         self.timeout = timeout
         self.limit = limit
@@ -58,6 +61,7 @@ class SingleEntry(Entry):
         try:
             process = subprocess.run(
                 self.script,
+                cwd=self.work_dir,
                 timeout=(self.timeout or self.default_timeout),
                 shell=True,
                 capture_output=True,
@@ -106,6 +110,7 @@ class SingleEntry(Entry):
         return {
             "type": "single",
             "id": self.id,
+            "work_dir": str(self.work_dir),
             "script": self.script,
             "timeout": self.timeout,
             "limit": self.limit,

sia/iteration_parser.py

@@ -1,6 +1,6 @@
-from datetime import datetime
+from pathlib import Path
-import xml.etree.ElementTree as ET
 from typing import List
+import xml.etree.ElementTree as ET
 
 from .entry import Entry
 from .entry.background_entry import BackgroundEntry
@@ -15,7 +15,7 @@ class IterationParser:
     """Parses iteration XML files into entries"""
 
     @staticmethod
-    def parse_iteration(content: str, io_buffer) -> tuple[str, str, List[Entry]]:
+    def parse_iteration(content: str, work_dir: Path, io_buffer) -> tuple[str, str, List[Entry]]:
         """
         Parse iteration XML content into context, response and entries
         
@@ -37,7 +37,7 @@ class IterationParser:
         entries = []
         for elem in context:
             if elem.tag == "background":
-                entries.append(IterationParser._parse_background(elem))
+                entries.append(IterationParser._parse_background(elem, work_dir))
             elif elem.tag == "parse_error": 
                 entries.append(IterationParser._parse_parse_error(elem))
             elif elem.tag == "read_stdin":
@@ -45,19 +45,20 @@ class IterationParser:
             elif elem.tag == "reasoning":
                 entries.append(IterationParser._parse_reasoning(elem))
             elif elem.tag == "repeat":
-                entries.append(IterationParser._parse_repeat(elem))
+                entries.append(IterationParser._parse_repeat(elem, work_dir))
             elif elem.tag == "single":
-                entries.append(IterationParser._parse_single(elem))
+                entries.append(IterationParser._parse_single(elem, work_dir))
             elif elem.tag == "write_stdout":
                 entries.append(IterationParser._parse_write(elem, io_buffer))
 
         return entries
 
     @staticmethod
-    def _parse_background(elem: ET.Element) -> BackgroundEntry:
+    def _parse_background(elem: ET.Element, work_dir: Path) -> BackgroundEntry:
         entry = BackgroundEntry(
             id=elem.get("id"),
-            script=elem.text
+            script=elem.text,
+            work_dir=work_dir
         )
         if elem.get("exit_code"):
             entry.exit_code = int(elem.get("exit_code"))
@@ -97,9 +98,10 @@ class IterationParser:
         )
 
     @staticmethod
-    def _parse_repeat(elem: ET.Element) -> RepeatEntry:
+    def _parse_repeat(elem: ET.Element, work_dir: Path) -> RepeatEntry:
         entry = RepeatEntry(
             id=elem.get("id"),
+            work_dir=work_dir,
             script=elem.text,
             timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
             limit=int(elem.get("limit")) if elem.get("limit") else None
@@ -116,9 +118,10 @@ class IterationParser:
         return entry
 
     @staticmethod
-    def _parse_single(elem: ET.Element) -> SingleEntry:
+    def _parse_single(elem: ET.Element, work_dir: Path) -> SingleEntry:
         entry = SingleEntry(
             id=elem.get("id"),
+            work_dir=work_dir,
             script=elem.text,
             timeout=float(elem.get("timeout")) if elem.get("timeout") else None,
             limit=int(elem.get("limit")) if elem.get("limit") else None

sia/response_parser.py

@@ -1,4 +1,5 @@
 from datetime import datetime
+from pathlib import Path
 import xml.etree.ElementTree as ET
 from typing import Union
 
@@ -30,13 +31,15 @@ class ResponseParser:
         io_buffer: Buffer to use for IO operations
     """
     
-    def __init__(self, io_buffer: IOBuffer):
+    def __init__(self, work_dir: Path, io_buffer: IOBuffer):
         """
         Initialize parser with IO buffer.
         
         Args:
+            work_dir: Workdir for the scripts
             io_buffer: Buffer to use for IO operations
         """
+        self._work_dir = work_dir
         self._io_buffer = io_buffer
     
     @property
@@ -87,7 +90,7 @@ class ResponseParser:
             elif root.tag == 'background':
                 if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '':
                     return ParseErrorEntry(entry_id, xml, "Background entry requires (only) script content")
-                return BackgroundEntry(entry_id, root.text)
+                return BackgroundEntry(entry_id, self._work_dir, root.text)
                 
             elif root.tag == 'repeat':
                 if len(root) != 0 or root.text is None or root.text.strip() == '':
@@ -98,7 +101,7 @@ class ResponseParser:
                 limit = root.get('limit')
                 timeout = float(timeout) if timeout is not None else None
                 limit = int(limit) if limit is not None else None
-                return RepeatEntry(entry_id, root.text, timeout, limit)
+                return RepeatEntry(entry_id, self._work_dir, root.text, timeout, limit)
                 
             elif root.tag == 'single':
                 if len(root) != 0 or root.text is None or root.text.strip() == '':
@@ -109,7 +112,7 @@ class ResponseParser:
                 limit = root.get('limit')
                 timeout = float(timeout) if timeout is not None else None
                 limit = int(limit) if limit is not None else None
-                return SingleEntry(entry_id, root.text, timeout, limit)
+                return SingleEntry(entry_id, self._work_dir, root.text, timeout, limit)
 
             elif root.tag == 'reasoning':
                 if len(root) != 0 or root.attrib or root.text is None or root.text.strip() == '':

sia/web/api.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from aiohttp import web
 import json
 import asyncio
@@ -13,12 +14,14 @@ from ..working_memory import WorkingMemory
 class Api:
     def __init__(
             self,
+            work_dir: Path,
             app: web.Application,
             agent: WebAgent,
             io_buffer: WebIOBuffer,
             working_memory: WorkingMemory,
             auto_approver: AutoApprover
         ):
+        self._work_dir = work_dir
         self._app = app
         self._agent = agent
         self._working_memory = working_memory
@@ -275,7 +278,7 @@ class Api:
         if not content:
             return web.Response(status=400, text="Missing content in request body")
             
-        entries = IterationParser.parse_iteration(content, self._io_buffer)
+        entries = IterationParser.parse_iteration(content, self._work_dir, self._io_buffer)
         
         for entry in entries:
             self._working_memory.add_entry(entry)

tools/itb/bin/itb_screenshot

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+import sys
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.remote.webelement import WebElement
+import xml.etree.ElementTree as ET
+from typing import Dict, List, Set, Optional
+from dataclasses import dataclass
+
+@dataclass
+class DOMNode:
+    element: WebElement
+    children: List['DOMNode']
+    own_text: str
+    total_text: str
+    tag: str
+    location: Dict[str, int]
+    size: Dict[str, int]
+    is_interactive: bool
+    attributes: Dict[str, str]
+
+class DOMTree:
+    def __init__(self, root_element: WebElement):
+        self.root = self._build_tree(root_element)
+        
+    def _get_element_own_text(self, element: WebElement) -> str:
+        script = """
+            let element = arguments[0];
+            let walker = document.createTreeWalker(
+                element,
+                NodeFilter.SHOW_TEXT,
+                {
+                    acceptNode: function(node) {
+                        return node.parentElement === element ? 
+                            NodeFilter.FILTER_ACCEPT : 
+                            NodeFilter.FILTER_REJECT;
+                    }
+                }
+            );
+            let text = '';
+            let node;
+            while (node = walker.nextNode()) {
+                text += node.textContent;
+            }
+            return text.trim();
+        """
+        return element.parent.execute_script(script, element)
+        
+    def _is_interactive(self, element: WebElement) -> bool:
+        return (
+            element.tag_name in {"button", "a", "input", "select", "textarea"} or
+            element.get_attribute("role") in {"button", "link", "textbox"} or
+            element.get_attribute("onclick") is not None or
+            element.get_attribute("contenteditable") == "true"
+        )
+        
+    def _get_attributes(self, element: WebElement) -> Dict[str, str]:
+        attrs = {}
+        for attr in ['placeholder', 'aria-label', 'alt', 'title', 'role']:
+            if value := element.get_attribute(attr):
+                attrs[attr] = value
+        return attrs
+        
+    def _build_tree(self, element: WebElement) -> DOMNode:
+        children = [
+            self._build_tree(child) 
+            for child in element.find_elements(By.XPATH, "./*")
+        ]
+        
+        own_text = self._get_element_own_text(element)
+        total_text = own_text + " " + " ".join(
+            child.total_text for child in children
+        )
+        total_text = " ".join(total_text.split())
+        
+        return DOMNode(
+            element=element,
+            children=children,
+            own_text=own_text,
+            total_text=total_text,
+            tag=element.tag_name,
+            location=element.location,
+            size=element.size,
+            is_interactive=self._is_interactive(element),
+            attributes=self._get_attributes(element)
+        )
+
+class ScreenshotGenerator:
+    IGNORED_TAGS = {'script', 'style', 'meta', 'link', 'noscript'}
+    
+    def __init__(self, debug_port: int):
+        options = webdriver.ChromeOptions()
+        options.add_experimental_option("debuggerAddress", f"127.0.0.1:{debug_port}")
+        self.driver = webdriver.Chrome(options=options)
+        
+    def get_viewport_metrics(self) -> Dict[str, int]:
+        return {
+            "scroll_x": self.driver.execute_script("return window.pageXOffset;"),
+            "scroll_y": self.driver.execute_script("return window.pageYOffset;"),
+            "width": self.driver.execute_script("return document.documentElement.clientWidth;"),
+            "height": self.driver.execute_script("return document.documentElement.clientHeight;"),
+            "max_scroll_y": self.driver.execute_script(
+                "return Math.max(document.documentElement.scrollHeight - window.innerHeight, 0);"
+            )
+        }
+        
+    def should_process_node(self, node: DOMNode) -> bool:
+        # Skip invisible elements
+        if not node.element.is_displayed():
+            return False
+            
+        # Skip script and style elements
+        if node.tag in self.IGNORED_TAGS:
+            return False
+
+        return True
+        
+    def create_xml_element(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> Optional[ET.Element]:
+        attrs = {
+            "x": str(node.location["x"]),
+            "y": str(node.location["y"]),
+            "width": str(node.size["width"]),
+            "height": str(node.size["height"])
+        }
+        
+        if node.is_interactive:
+            element = ET.Element("interactive", attrs)
+            child = ET.SubElement(element, node.tag)
+            for attr, value in node.attributes.items():
+                child.set(attr, value)
+            if node.own_text:
+                child.text = node.own_text
+            return element
+            
+        element = ET.Element(node.tag, attrs)
+        if node.own_text:
+            element.text = node.own_text
+        return element
+        
+    def process_tree(self, node: DOMNode, parent_element: Optional[ET.Element] = None) -> List[ET.Element]:
+        if not self.should_process_node(node):
+            return []
+            
+        current_element = self.create_xml_element(node, parent_element)
+        if not current_element:
+            return []
+            
+        for child in node.children:
+            child_elements = self.process_tree(child, current_element)
+            for child_element in child_elements:
+                current_element.append(child_element)
+                
+        return [current_element]
+        
+    def generate_output(self) -> ET.Element:
+        metrics = self.get_viewport_metrics()
+        viewport = ET.Element("viewport", {
+            "scroll_x": str(metrics["scroll_x"]),
+            "scroll_y": str(metrics["scroll_y"]),
+            "width": str(metrics["width"]),
+            "height": str(metrics["height"]),
+            "max_scroll_y": str(metrics["max_scroll_y"])
+        })
+        
+        dom_tree = DOMTree(self.driver.find_element(By.TAG_NAME, "body"))
+        for element in self.process_tree(dom_tree.root):
+            viewport.append(element)
+            
+        return viewport
+
+def main(debug_port: int):
+    generator = ScreenshotGenerator(debug_port)
+    output = generator.generate_output()
+    print(ET.tostring(output, encoding="unicode", method="xml"))
+
+if __name__ == "__main__":
+    if len(sys.argv) != 2:
+        print("Usage: itb_screenshot <debug_port>")
+        sys.exit(1)
+    main(int(sys.argv[1]))

tools/itb/bin/itb_start

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+import random
+import subprocess
+import sys
+
+def start(url):
+    debug_port = random.randint(9000, 9999)
+    chrome_cmd = [
+        '/usr/bin/google-chrome',
+        '--headless=new',
+        '--no-sandbox',
+        '--remote-debugging-port=' + str(debug_port),
+        url
+    ]
+    
+    process = subprocess.Popen(
+        chrome_cmd,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+        start_new_session=True
+    )
+    
+    print({
+        "pid": process.pid,
+        "debug_port": debug_port
+    })
+
+
+if __name__ == '__main__':
+    if len(sys.argv) != 2:
+        print("Usage: itb_start <url>")
+        sys.exit(1)
+    start(sys.argv[1])

tools/itb/itb.egg-info/PKG-INFO

@@ -0,0 +1,11 @@
+Metadata-Version: 2.1
+Name: itb
+Version: 0.1.0
+Summary: UNKNOWN
+Home-page: UNKNOWN
+License: UNKNOWN
+Platform: UNKNOWN
+Provides-Extra: dev
+
+UNKNOWN
+

tools/itb/itb.egg-info/SOURCES.txt

@@ -0,0 +1,16 @@
+README.md
+setup.py
+bin/itb_click
+bin/itb_forms
+bin/itb_input
+bin/itb_links
+bin/itb_navigate
+bin/itb_refresh
+bin/itb_screenshot
+bin/itb_scroll
+bin/itb_start
+itb.egg-info/PKG-INFO
+itb.egg-info/SOURCES.txt
+itb.egg-info/dependency_links.txt
+itb.egg-info/requires.txt
+itb.egg-info/top_level.txt

tools/itb/itb.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

tools/itb/itb.egg-info/requires.txt

@@ -0,0 +1,10 @@
+beautifulsoup4>=4.9.0
+click>=8.0.0
+selenium>=4.0.0
+webdriver-manager>=3.8.0
+
+[dev]
+black>=22.0.0
+flake8>=4.0.0
+pytest-cov>=4.0.0
+pytest>=7.0.0

tools/itb/itb.egg-info/top_level.txt

@@ -0,0 +1 @@
+

tools/itb/requirements.txt

@@ -0,0 +1,8 @@
+selenium>=4.0.0
+webdriver-manager>=3.8.0
+click>=8.0.0
+beautifulsoup4>=4.9.0
+pytest>=7.0.0
+pytest-cov>=4.0.0
+black>=22.0.0
+flake8>=4.0.0

tools/itb/result.social.html

@@ -0,0 +1,38 @@
+<viewport scroll_x="0" scroll_y="0" width="780" height="441" max_scroll_y="1200">
+    <nav x="0" y="0" width="100%" height="60">
+        <h1>SocialConnect</h1>
+        <interactive x="16" y="0" width="240" height="40">
+            <input placeholder="Search posts..." />
+        </interactive>
+    </nav>
+    
+    <div class="post" x="20" y="80" width="680" height="400">
+        <div class="username" x="72" y="96" width="200" height="20">Alice Johnson</div>
+        <div class="timestamp" x="72" y="116" width="200" height="16">2 hours ago</div>
+        
+        <div class="post-content" x="36" y="148" width="648" height="60">
+            Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 #MorningHike #Nature #Wellness
+        </div>
+        
+        <img x="36" y="220" width="648" height="400" alt="Sunrise view from mountain top" />
+        
+        <div class="actions" x="36" y="632" width="648" height="40">
+            <interactive x="36" y="632" width="216" height="40">
+                <button>Like</button>
+            </interactive>
+            <interactive x="252" y="632" width="216" height="40">
+                <button>Comment</button>
+            </interactive>
+            <interactive x="468" y="632" width="216" height="40">
+                <button>Share</button>
+            </interactive>
+        </div>
+        
+        <div class="comment" x="44" y="684" width="632" height="60">
+            <div class="username" x="96" y="684" width="200" height="20">Bob Smith</div>
+            <div class="comment-content" x="96" y="704" width="580" height="40">
+                Beautiful view! Which trail is this?
+            </div>
+        </div>
+    </div>
+</viewport>

tools/itb/result.test.html

@@ -0,0 +1,32 @@
+<viewport scroll_y="0" scroll_x="0" height="800" width="1024">
+  <header class="sticky" y="0" x="0" width="1024" height="50">
+    <form id="search">
+      <input type="text" placeholder="Search..." aria-label="Search box" y="10" x="10" width="200" height="30" />
+      <button type="submit" y="10" x="220" width="30" height="30">🔍</button>
+    </form>
+  </header>
+  
+  <section class="scrollable" y="50" x="0" width="1024" height="200" scroll_y="0" max_scroll="400">
+    <heading level="2">Scrollable Section</heading>
+    Lorem ipsum dolor sit amet, consectetur adipiscing elit... [first paragraph]
+  </section>
+
+  <interactive y="260" x="10" width="100" height="30">
+    <button onclick="showModal()">Show Modal</button>
+  </interactive>
+  
+  <interactive y="260" x="120" width="120" height="30">
+    <link href="#">JavaScript Link</link>
+  </interactive>
+  
+  <interactive y="260" x="250" width="100" height="30">
+    <button role="button" tabindex="0">ARIA Button</button>
+  </interactive>
+
+  <table y="300" x="0" width="1024">
+    <cell>Cell with mixed formatting</cell>
+    <cell>Cell with list:
+- Item 1
+- Item 2</cell>
+  </table>
+</viewport>

tools/itb/setup.py

@@ -0,0 +1,32 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="itb",
+    version="0.1.0",
+    packages=find_packages(),
+    scripts=[
+        'bin/itb_start',
+        'bin/itb_screenshot',
+        'bin/itb_scroll',
+        'bin/itb_click',
+        'bin/itb_input',
+        'bin/itb_links',
+        'bin/itb_forms',
+        'bin/itb_navigate',
+        'bin/itb_refresh'
+    ],
+    install_requires=[
+        'selenium>=4.0.0',
+        'webdriver-manager>=3.8.0',
+        'click>=8.0.0',
+        'beautifulsoup4>=4.9.0'
+    ],
+    extras_require={
+        'dev': [
+            'pytest>=7.0.0',
+            'pytest-cov>=4.0.0',
+            'black>=22.0.0',
+            'flake8>=4.0.0'
+        ]
+    }
+)

tools/itb/social.html

@@ -0,0 +1,178 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <title>SocialConnect</title>
+    <style>
+        body {
+            margin: 0;
+            font-family: Arial, sans-serif;
+            background: #f0f2f5;
+        }
+        .navbar {
+            position: fixed;
+            top: 0;
+            width: 100%;
+            height: 60px;
+            background: white;
+            box-shadow: 0 2px 4px rgba(0,0,0,0.1);
+            z-index: 100;
+            display: flex;
+            align-items: center;
+            padding: 0 16px;
+        }
+        .search-box {
+            margin-left: 16px;
+            padding: 8px;
+            border-radius: 20px;
+            border: 1px solid #ddd;
+            width: 240px;
+        }
+        .main-content {
+            margin-top: 80px;
+            margin-left: auto;
+            margin-right: auto;
+            max-width: 680px;
+            padding: 20px;
+        }
+        .post {
+            background: white;
+            border-radius: 8px;
+            padding: 16px;
+            margin-bottom: 16px;
+            box-shadow: 0 1px 2px rgba(0,0,0,0.1);
+        }
+        .post-header {
+            display: flex;
+            align-items: center;
+            margin-bottom: 12px;
+        }
+        .profile-pic {
+            width: 40px;
+            height: 40px;
+            border-radius: 50%;
+            margin-right: 12px;
+            background: #ddd;
+        }
+        .post-meta {
+            flex-grow: 1;
+        }
+        .username {
+            font-weight: bold;
+            color: #1a1a1a;
+        }
+        .timestamp {
+            color: #65676b;
+            font-size: 13px;
+        }
+        .post-content {
+            margin-bottom: 12px;
+            line-height: 1.5;
+        }
+        .post-image {
+            width: 100%;
+            max-height: 400px;
+            object-fit: cover;
+            border-radius: 8px;
+            margin-bottom: 12px;
+        }
+        .post-actions {
+            display: flex;
+            border-top: 1px solid #ddd;
+            padding-top: 12px;
+        }
+        .action-button {
+            flex: 1;
+            text-align: center;
+            padding: 8px;
+            border: none;
+            background: none;
+            cursor: pointer;
+            color: #65676b;
+            font-weight: 600;
+        }
+        .action-button:hover {
+            background: #f2f2f2;
+            border-radius: 4px;
+        }
+        .comment-section {
+            margin-top: 12px;
+            padding-top: 12px;
+            border-top: 1px solid #ddd;
+        }
+        .comment {
+            display: flex;
+            margin-bottom: 8px;
+        }
+        .comment-content {
+            background: #f0f2f5;
+            padding: 8px 12px;
+            border-radius: 18px;
+            margin-left: 8px;
+            flex-grow: 1;
+        }
+    </style>
+</head>
+<body>
+    <nav class="navbar">
+        <h1>SocialConnect</h1>
+        <input type="text" class="search-box" placeholder="Search posts..." aria-label="Search posts">
+    </nav>
+
+    <main class="main-content">
+        <!-- Generate 20 posts with varying content -->
+        <div class="post">
+            <div class="post-header">
+                <div class="profile-pic"></div>
+                <div class="post-meta">
+                    <div class="username">Alice Johnson</div>
+                    <div class="timestamp">2 hours ago</div>
+                </div>
+            </div>
+            <div class="post-content">
+                Just finished my morning hike! The sunrise was absolutely breathtaking today. 🌄 
+                #MorningHike #Nature #Wellness
+            </div>
+            <img src="data:image/svg+xml,<svg xmlns='http://www.w3.org/2000/svg' width='648' height='400'><rect width='100%' height='100%' fill='%23ddd'/></svg>" alt="Sunrise view from mountain top" class="post-image">
+            <div class="post-actions">
+                <button class="action-button">Like</button>
+                <button class="action-button">Comment</button>
+                <button class="action-button">Share</button>
+            </div>
+            <div class="comment-section">
+                <div class="comment">
+                    <div class="profile-pic"></div>
+                    <div class="comment-content">
+                        <div class="username">Bob Smith</div>
+                        Beautiful view! Which trail is this?
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <!-- Repeat similar structure with different content 20 times -->
+        <!-- This is post 2 -->
+        <div class="post">
+            <div class="post-header">
+                <div class="profile-pic"></div>
+                <div class="post-meta">
+                    <div class="username">Sarah Chen</div>
+                    <div class="timestamp">3 hours ago</div>
+                </div>
+            </div>
+            <div class="post-content">
+                Just launched my new portfolio website! 🚀 Excited to share my latest projects with everyone.
+                Check it out and let me know what you think! #WebDevelopment #Portfolio #Coding
+            </div>
+            <div class="post-actions">
+                <button class="action-button">Like</button>
+                <button class="action-button">Comment</button>
+                <button class="action-button">Share</button>
+            </div>
+        </div>
+
+        <!-- Continue with more posts... -->
+        <!-- Posts 3-20 would follow similar pattern with varied content -->
+
+    </main>
+</body>
+</html>

tools/itb/test.html

@@ -0,0 +1,90 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <style>
+        .scrollable-div {
+            height: 200px;
+            overflow-y: scroll;
+            border: 1px solid black;
+        }
+        .sticky-header {
+            position: sticky;
+            top: 0;
+            background: white;
+        }
+        .modal {
+            position: fixed;
+            top: 50%;
+            left: 50%;
+            transform: translate(-50%, -50%);
+            z-index: 100;
+        }
+    </style>
+</head>
+<body>
+    <!-- Sticky header with form -->
+    <header class="sticky-header">
+        <form id="search">
+            <input type="text" placeholder="Search..." aria-label="Search box">
+            <button type="submit">🔍</button>
+        </form>
+    </header>
+
+    <!-- Nested scrollable content -->
+    <div class="scrollable-div">
+        <h2>Scrollable Section</h2>
+        <p>
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nam a bibendum purus, id accumsan lorem. Ut quis diam semper, rutrum justo in, ornare urna. Proin vel consectetur odio. Donec quis odio ut erat lobortis aliquam. Vestibulum vitae tellus interdum, hendrerit odio id, varius urna. Sed nec est tempor, placerat lectus sit amet, aliquam mauris. Duis elementum risus vitae nunc mollis ullamcorper. Donec sodales ac neque at aliquam. Duis eu risus ex. Fusce neque erat, egestas nec placerat in, dapibus quis magna. Class aptent taciti sociosqu ad litora torquent per conubia nostra, per inceptos himenaeos. Ut faucibus nunc tellus, vel consequat dolor suscipit vel. Sed sapien erat, rutrum non dignissim nec, egestas at augue. Nunc interdum lacinia urna a euismod. Integer ullamcorper sit amet sapien non auctor.
+        </p>
+        <p>
+Nam vitae bibendum nulla, ut tempus enim. Aenean eu iaculis metus, vel rhoncus justo. Sed vel interdum sem. Praesent congue erat sit amet ultrices molestie. Pellentesque pellentesque odio a ullamcorper facilisis. Sed at viverra magna. Aliquam posuere dolor tortor, non aliquam orci bibendum eu. Sed efficitur feugiat congue. Vivamus elementum in metus lacinia gravida.
+        </p>
+        <p>
+Fusce aliquam, erat ac consequat eleifend, massa neque posuere urna, vel sagittis tortor neque hendrerit augue. Nulla mattis felis non efficitur hendrerit. Pellentesque viverra elementum enim ornare pellentesque. Donec mollis eget leo non congue. Aliquam vel iaculis odio. Mauris eget neque suscipit, tempus mi quis, porta quam. Integer non tempor leo. Pellentesque magna nisl, fermentum sit amet orci ut, posuere dictum erat. Vivamus a lacinia lacus, eleifend ultrices velit. Etiam est arcu, dictum at sapien efficitur, fringilla tincidunt sem. In nec lacus quis diam placerat sollicitudin. Vivamus scelerisque commodo velit, id lacinia orci volutpat quis. Nulla facilisi. Nam nec quam quis lorem vulputate auctor quis a dui.
+        </p>
+        <p>
+Curabitur a erat nisi. Fusce lacinia, enim quis mollis placerat, eros nulla blandit turpis, id feugiat purus enim ut diam. Etiam et tincidunt turpis, in scelerisque lectus. Nunc vulputate at velit sit amet tincidunt. Cras maximus ultrices enim, sit amet placerat nisi tristique eget. Sed ac neque sed quam laoreet pellentesque nec vel lorem. Nunc sollicitudin arcu ac quam tempor bibendum. Phasellus sollicitudin tortor at porta lobortis. Aenean venenatis ultrices neque ut scelerisque. Nullam aliquam fringilla varius. Etiam eget nunc leo. Proin ornare tempus libero eget molestie. Praesent id ex sit amet ante semper dictum. Mauris ac libero at ipsum dapibus pretium quis at sapien. Praesent ac luctus lacus. Praesent id varius nisl, et luctus ex.
+        </p>
+        <iframe src="about:blank" title="Embedded content">
+            <p>
+Sed ullamcorper sapien non nunc venenatis rutrum. Nulla mollis eu augue non mollis. Maecenas molestie congue libero. Nulla non mi sem. Cras sed ipsum et orci porta posuere et in metus. Vivamus gravida leo a risus interdum, et aliquet neque pretium. Duis a mauris nec erat tempus lacinia. Cras a justo vel dui varius mattis sed ac neque. Phasellus blandit massa sed elit consectetur placerat. Sed aliquet et lacus nec posuere. Donec mattis dolor id tortor facilisis consectetur. Phasellus sit amet magna non nisl pretium semper. Sed viverra, mauris vitae ultricies interdum, libero felis aliquet massa, at vehicula elit lacus a diam. Aliquam sed erat et justo tempor mattis eget et neque. Ut fermentum magna ut lorem sodales condimentum. Vestibulum elementum nec urna sed suscipit. 
+            </p>
+        </iframe>
+        <div contenteditable="true">
+Curabitur a erat nisi. Fusce lacinia, enim quis mollis placerat, eros nulla blandit turpis, id feugiat purus enim ut diam. Etiam et tincidunt turpis, in scelerisque lectus. Nunc vulputate at velit sit amet tincidunt. Cras maximus ultrices enim, sit amet placerat nisi tristique eget. Sed ac neque sed quam laoreet pellentesque nec vel lorem. Nunc sollicitudin arcu ac quam tempor bibendum. Phasellus sollicitudin tortor at porta lobortis. Aenean venenatis ultrices neque ut scelerisque. Nullam aliquam fringilla varius. Etiam eget nunc leo. Proin ornare tempus libero eget molestie. Praesent id ex sit amet ante semper dictum. Mauris ac libero at ipsum dapibus pretium quis at sapien. Praesent ac luctus lacus. Praesent id varius nisl, et luctus ex.
+        </div>
+    </div>
+
+    <!-- Interactive elements with dynamic content -->
+    <div>
+        <button onclick="showModal()">Show Modal</button>
+        <a href="#" onclick="return false;">JavaScript Link</a>
+        <div role="button" tabindex="0">ARIA Button</div>
+    </div>
+
+    <!-- Modal with overlay -->
+    <div class="modal" style="display:none">
+        <h3>Modal Title</h3>
+        <form>
+            <select multiple>
+                <option>Choice 1</option>
+                <option>Choice 2</option>
+            </select>
+            <input type="range" min="0" max="100">
+        </form>
+    </div>
+
+    <!-- Complex text layout -->
+    <table>
+        <tr>
+            <td>Cell with <strong>mixed</strong> <em>formatting</em></td>
+            <td>Cell with list:
+                <ul>
+                    <li>Item 1</li>
+                    <li>Item 2</li>
+                </ul>
+            </td>
+        </tr>
+    </table>
+</body>
+</html>

web/src/components/App.jsx

@@ -251,6 +251,8 @@ const App = () => {
     if (activeTab === Tabs.MEMORY) return;
     const state = llms[activeLlm];
     switch (state) {
+      case null:
+        break;
       case LlmState.NO_OUTPUT:
         setActiveTab(Tabs.CONTEXT);
         setShowDiff(false);