From d80171de1560738ae84996cc04de2305aade3367 Mon Sep 17 00:00:00 2001 From: geens Date: Fri, 1 Nov 2024 11:38:28 +0100 Subject: [PATCH] Response parser and split up class diagrams --- readme.md | 407 ++++++++++++++++++++++------------- sia/response_parser.py | 154 +++++++++++++ sia/system_metrics.py | 122 +++++++++++ test/llm_engine_test.py | 25 --- test/response_parser_test.py | 215 ++++++++++++++++++ test/system_metrics_test.py | 186 ++++++++++++++++ 6 files changed, 932 insertions(+), 177 deletions(-) create mode 100644 sia/response_parser.py create mode 100644 sia/system_metrics.py delete mode 100644 test/llm_engine_test.py create mode 100644 test/response_parser_test.py create mode 100644 test/system_metrics_test.py diff --git a/readme.md b/readme.md index bfb3c71..8480ef8 100644 --- a/readme.md +++ b/readme.md @@ -267,7 +267,100 @@ This architecture allows for: - Human-in-the-loop operation for testing and development - Collection of human feedback for reinforcement learning -### Standard Agent Flow +### Diagrams + +#### Core classes + +```mermaid +classDiagram + class SystemMetrics { + +SystemMetrics(sample_interval float) + +generate_context(context_usage float) ElementTree + +stop() void + -monitor_loop() void + } + + class LLMEngine { + +LLMEngine(model_path str) + +set_model_path(model_path str) void + +infer(system_prompt str, main_context str) Iterator~str~ + } + + class BaseAgent { + <> + #working_memory: WorkingMemory + #metrics: SystemMetrics + #llm: LLMEngine + #parser: ResponseParser + #validator: XMLValidator + #io_buffer: IOBuffer + + #compile_context() str + } + + class WorkingMemory { + -entries: List~Entry~ + + +WorkingMemory() + +add_entry(entry Entry) void + +remove_entry(id str) void + +clear() void + +get_entry(id str) Optional~Entry~ + +get_entries() List~Entry~ + +get_entries_count() int + +get_entries_by_type(type Type) List~Entry~ + +update() void + +generate_context() List~ElementTree~ + } + + class XMLValidator { + +XMLValidator(schema str) + +validate(xml str) Optional~str~ + +get_valid_root_elements() Set~str~ + } + + class ResponseParser { + -io_buffer: IOBuffer + + +ResponseParser(io_buffer IOBuffer) + +parse(xml str) Union~Command, Entry~ + } + + class Entry { + <> + +id: str + +timestamp: datetime + + +Entry(id str, timestamp datetime) + +update() void* + +generate_context() ElementTree* + +cleanup() void* + } + + class IOBuffer { + <> + +read() str* + +write(content str) void* + +buffer_length() int* + } + + class Command { + <> + +execute(memory WorkingMemory) CommandResult* + } + + SystemMetrics "1" --* "1" BaseAgent + LLMEngine "1" --* "1" BaseAgent + XMLValidator "1" --* "1" BaseAgent + BaseAgent "1" *-- "1" IOBuffer + BaseAgent "1" *-- "1" WorkingMemory + BaseAgent "1" *-- "1" ResponseParser + WorkingMemory "1" *-- "*" Entry + ResponseParser ..> Entry + ResponseParser ..> Command +``` + +#### Standard Agent Flow ```mermaid stateDiagram-v2 @@ -287,7 +380,74 @@ stateDiagram-v2 } ``` -### Web Agent Flow +#### Web Agent + +```mermaid +classDiagram + class BaseAgent { + <> + #working_memory: WorkingMemory + #metrics: SystemMetrics + #llm: LLMEngine + #parser: ResponseParser + #validator: XMLValidator + #io_buffer: IOBuffer + + #compile_context() str + } + + class StandardAgent { + +StandardAgent(model_path str, system_prompt str, action_schema str) + +run() void + } + + class WebAgent { + -current_state: WebAgentState + + +WebAgent(model_path str, system_prompt str, action_schema str) + +get_current_state() WebAgentState + +proceed() void + +add_state_change_handler(handler func) void + } + + class WebAgentState { + <> + UPDATE + CONTEXT_APPROVAL + INFERENCE + RESPONSE_APPROVAL + } + + class WebServer { + -agent: WebAgent + -clients: List~WebSocket~ + + +WebServer(agent WebAgent) + -broadcast_state_change() void + } + + class WebIOBuffer { + -stdin_buffer: str + -stdout_buffer: str + + +WebIOBuffer() + +read() str + +write(content str) void + +buffer_length() int + +append_stdin(content str) void + +get_stdout() str + +clear_stdout() void + } + + BaseAgent <|-- WebAgent + BaseAgent <|-- StandardAgent + + WebServer "1" *-- "1" WebIOBuffer + WebServer "1" *-- "1" WebAgent + WebAgent "1" *-- "1" WebAgentState +``` + +#### Web Agent Flow ```mermaid stateDiagram-v2 @@ -314,226 +474,169 @@ stateDiagram-v2 } ``` -### Class Diagram +#### Entry classes ```mermaid classDiagram - direction LR - class BaseAgent { - <> - #working_memory WorkingMemory - #metrics SystemMetrics - #llm LLMEngine - #parser ResponseParser - #validator XMLValidator - #io_buffer IOBuffer - #compile_context() str - } - - class LLMEngine { - +LLMEngine(model_path str) - +set_model_path(model_path str) - +inference(context str) Iterator~str~ - } - - class StandardAgent { - +run() - } - - class WebAgent { - -current_state WebAgentState - +get_current_state() WebAgentState - +proceed() - +add_state_change_handler(handler func()) - } - - class WebAgentState { - <> - UPDATE - CONTEXT_APPROVAL - INFERENCE - RESPONSE_APPROVAL - } - - class WorkingMemory { - +WorkingMemory() - -entries List~Entry~ - +add_entry(entry Entry) - +remove_entry(id str) - +update() - +generate_context() List~ElementTree~ - } - - class SystemMetrics { - +generate_context(entries List~ElementTree~) ElementTree - } - - class XMLValidator { - -schema ElementTree - +XMLValidator(schema str) - +validate(xml str) Optional~str~ - } - - class ResponseParser { - -io_buffer IOBuffer - +ResponseParser(io_buffer IOBuffer) - +parse(xml str) Command | Entry - } - class Entry { <> - +id str - +timestamp datetime + +id: str + +timestamp: datetime + +Entry(id str, timestamp datetime) - +update()* + +update() void* +generate_context() ElementTree* + +cleanup() void* } class SingleShotEntry { - +script str - +stdout str - +stderr str - +exit_code int + +script: str + +stdout: str + +stderr: str + +exit_code: Optional~int~ + +SingleShotEntry(script str, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree } class RepeatEntry { - +script str - +stdout str - +stderr str - +exit_code int + +script: str + +stdout: str + +stderr: str + +exit_code: Optional~int~ + +RepeatEntry(script str, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree } class BackgroundEntry { - +script str - +stdout str - +stderr str - +process Process + +script: str + +stdout: str + +stderr: str + +process: Optional~Process~ + -accumulated_stdout: str + -accumulated_stderr: str + -exit_code: Optional~int~ + +BackgroundEntry(script str, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree + +cleanup() void } class ReasoningEntry { - +content str + +content: str + +ReasoningEntry(content str, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree } class ParseErrorEntry { - +content str - +error str + +content: str + +error: str + +ParseErrorEntry(content str, error str, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree } class ReadEntry { - +content str + +content: str + +io_buffer: IOBuffer + +ReadEntry(io_buffer IOBuffer, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree } class WriteEntry { - +content str + +content: str + +io_buffer: IOBuffer + +WriteEntry(content str, io_buffer IOBuffer, id str, timestamp datetime) - +update() + +update() void +generate_context() ElementTree } + ReasoningEntry --|> Entry + ParseErrorEntry --|> Entry + ReadEntry --|> Entry + Entry <|-- WriteEntry + Entry <|-- SingleShotEntry + Entry <|-- RepeatEntry + Entry <|-- BackgroundEntry +``` + +#### IO Buffer classes + +```mermaid +classDiagram class IOBuffer { <> +read() str* - +write(content str)* + +write(content str) void* +buffer_length() int* } class StandardIOBuffer { +StandardIOBuffer() +read() str - +write(content str) + +write(content str) void +buffer_length() int } class WebIOBuffer { - -stdin_buffer str - -stdout_buffer str + -stdin_buffer: str + -stdout_buffer: str + +WebIOBuffer() +read() str - +write(content str) + +write(content str) void +buffer_length() int - +append_stdin(content str) + +append_stdin(content str) void +get_stdout() str - +clear_stdout() + +clear_stdout() void + } + + IOBuffer <|.. WebIOBuffer + IOBuffer <|.. StandardIOBuffer +``` + +#### Command classes + +```mermaid +classDiagram + direction LR + class Command { + <> + +execute(memory WorkingMemory) CommandResult* + } + + class DeleteCommand { + +DeleteCommand(id str) + +execute(memory WorkingMemory) CommandResult + } + + class StopCommand { + +StopCommand() + +execute(memory WorkingMemory) CommandResult } class CommandResult { - +should_stop bool - +success bool - +message str - +CommandResult(should_stop bool, success bool, message str) + +message: str + +success: bool + +should_stop: bool + + +CommandResult(message str, success bool, should_stop bool) +static success() CommandResult +static failure(message str) CommandResult +static stop() CommandResult } - class Command { - <> - +execute(memory &WorkingMemory) CommandResult* - } - - class DeleteCommand { - +id str - +DeleteCommand(id str) - +execute(memory &WorkingMemory) CommandResult - } - - class StopCommand { - +execute(memory &WorkingMemory) CommandResult - } - - class WebServer { - -agent WebAgent - -clients List~WebSocket~ - +WebServer(agent WebAgent) - -broadcast_state_change() - } - - BaseAgent <|-- WebAgent - BaseAgent <|-- StandardAgent - BaseAgent "1" *-- "1" IOBuffer - BaseAgent "1" *-- "1" WorkingMemory - BaseAgent "1" *-- "1" SystemMetrics - BaseAgent "1" *-- "1" XMLValidator - BaseAgent "1" *-- "1" LLMEngine - BaseAgent "1" *-- "1" ResponseParser - - WebServer "1" *-- "1" WebIOBuffer - WebServer "1" *-- "1" WebAgent - - WebAgent "1" *-- "1" WebAgentState - - WorkingMemory "1" *-- "*" Entry - - Entry <|-- ReasoningEntry - Entry <|-- ParseErrorEntry - Entry <|-- ReadEntry - Entry <|-- WriteEntry - Entry <|-- SingleShotEntry - Entry <|-- RepeatEntry - Entry <|-- BackgroundEntry - Command <|-- DeleteCommand Command <|-- StopCommand Command -- CommandResult - - IOBuffer <|.. WebIOBuffer - IOBuffer <|.. StandardIOBuffer ``` \ No newline at end of file diff --git a/sia/response_parser.py b/sia/response_parser.py new file mode 100644 index 0000000..52e954d --- /dev/null +++ b/sia/response_parser.py @@ -0,0 +1,154 @@ +from datetime import datetime +import uuid +import xml.etree.ElementTree as ET +from typing import Union + +from .command import Command +from .delete_command import DeleteCommand +from .stop_command import StopCommand + +from .entry import Entry +from .background_entry import BackgroundEntry +from .parse_error_entry import ParseErrorEntry +from .read_entry import ReadEntry +from .reasoning_entry import ReasoningEntry +from .repeat_entry import RepeatEntry +from .single_shot_entry import SingleShotEntry +from .write_entry import WriteEntry + +class ResponseParser: + """ + Parses XML responses from the LLM into commands or entries. + + The parser validates the XML structure and converts it into the appropriate + Command or Entry object based on the root element tag. Invalid input results + in ParseErrorEntry objects rather than raising exceptions. + + Attributes: + io_buffer: Buffer to use for IO operations + """ + + def __init__(self, io_buffer): + """ + Initialize parser with IO buffer. + + Args: + io_buffer: Buffer to use for IO operations + """ + self.io_buffer = io_buffer + + def _extract_content(self, xml: str, root: ET.Element) -> str: + """ + Extract content from XML, handling CDATA sections and preserving special characters. + + Args: + xml: Original XML string + root: Parsed XML element + + Returns: + str: Extracted content with proper handling of CDATA and special characters + """ + if root.text is None: + return "" + + # Find opening tag end + tag_name = root.tag + tag_start = xml.find('<' + tag_name) + tag_end = xml.find('>', tag_start) + if tag_end == -1: + return "" + + # Check for self-closing tag + if xml[tag_end-1] == '/': + return "" + + # Find closing tag start + close_tag = '' + close_start = xml.rfind(close_tag) + if close_start == -1: + return "" + + # Extract content + content = xml[tag_end+1:close_start] + + # Handle CDATA sections + if content.startswith(''): + content = content[9:-3] # Remove CDATA markers + + return content + + def parse(self, xml: str) -> Union[Command, Entry]: + """ + Parse XML response into a Command or Entry. + + Args: + xml: XML string to parse + + Returns: + Command or Entry based on the XML content + ParseErrorEntry for any parsing or validation errors + """ + entry_id = str(uuid.uuid4()) + timestamp = datetime.now() + + # First try to parse the XML + try: + root = ET.fromstring(xml.strip()) + except ET.ParseError as e: + return ParseErrorEntry(xml, f"Invalid XML: {str(e)}", entry_id, timestamp) + + # Extract content preserving special characters and handling CDATA + content = self._extract_content(xml, root) + + # Convert to appropriate type based on root tag + try: + if root.tag == 'delete': + # Extract target ID and create delete command + target_id = root.get('id') + if not target_id: + return ParseErrorEntry(xml, "Delete command missing required 'id' attribute", entry_id, timestamp) + return DeleteCommand(target_id) + + elif root.tag == 'stop': + # Create stop command + return StopCommand() + + elif root.tag == 'background': + # Create background script entry + if not content: + return ParseErrorEntry(xml, "Background entry missing script content", entry_id, timestamp) + return BackgroundEntry(content, entry_id, timestamp) + + elif root.tag == 'repeat': + # Create repeat script entry + if not content: + return ParseErrorEntry(xml, "Repeat entry missing script content", entry_id, timestamp) + return RepeatEntry(content, entry_id, timestamp) + + elif root.tag == 'single_shot': + # Create single shot script entry + if not content: + return ParseErrorEntry(xml, "Single shot entry missing script content", entry_id, timestamp) + return SingleShotEntry(content, entry_id, timestamp) + + elif root.tag == 'reasoning': + # Create reasoning entry + if not content: + return ParseErrorEntry(xml, "Reasoning entry missing content", entry_id, timestamp) + return ReasoningEntry(content, entry_id, timestamp) + + elif root.tag == 'read_stdin': + # Create read entry - no content required + return ReadEntry(self.io_buffer, entry_id, timestamp) + + elif root.tag == 'write_stdout': + # Create write entry + if not content: + return ParseErrorEntry(xml, "Write entry missing content", entry_id, timestamp) + return WriteEntry(content, self.io_buffer, entry_id, timestamp) + + else: + return ParseErrorEntry(xml, f"Unknown root element: {root.tag}", entry_id, timestamp) + + except Exception as e: + return ParseErrorEntry(xml, f"Error parsing response: {str(e)}", entry_id, timestamp) \ No newline at end of file diff --git a/sia/system_metrics.py b/sia/system_metrics.py new file mode 100644 index 0000000..6c3ebe6 --- /dev/null +++ b/sia/system_metrics.py @@ -0,0 +1,122 @@ +import threading +import time +from typing import Optional +import xml.etree.ElementTree as ET +import psutil + +class SystemMetrics: + """ + Tracks system metrics including CPU, GPU, memory and disk usage. + Maintains rolling averages for CPU and GPU between context generations. + + Attributes: + _stop_event: Threading event to signal monitor thread to stop + _monitor_thread: Background thread that collects usage data + _cpu_samples: List of CPU usage samples since last context + _gpu_samples: List of GPU usage samples since last context + _metrics_lock: Lock for thread-safe access to samples + """ + + def __init__(self, sample_interval: float = 0.1): + """ + Initialize system metrics monitoring. + + Args: + sample_interval: Seconds between usage samples + """ + self._stop_event = threading.Event() + self._cpu_samples = [] + self._gpu_samples = [] + self._metrics_lock = threading.Lock() + self._sample_interval = sample_interval + + # Start monitoring thread + self._monitor_thread = threading.Thread( + target=self._monitor_loop, + daemon=True + ) + self._monitor_thread.start() + + def _get_gpu_usage(self) -> Optional[float]: + """ + Get current GPU usage percentage. + Returns None if GPU monitoring not available. + """ + try: + import pynvml + pynvml.nvmlInit() + handle = pynvml.nvmlDeviceGetHandleByIndex(0) + info = pynvml.nvmlDeviceGetUtilizationRates(handle) + pynvml.nvmlShutdown() + return info.gpu + except: + return None + + def _monitor_loop(self): + """ + Background loop that collects system usage samples. + Runs until stop event is set. + """ + while not self._stop_event.is_set(): + with self._metrics_lock: + # Get CPU usage across all cores + self._cpu_samples.append(psutil.cpu_percent()) + + # Get GPU usage if available + gpu_usage = self._get_gpu_usage() + if gpu_usage is not None: + self._gpu_samples.append(gpu_usage) + + time.sleep(self._sample_interval) + + def generate_context(self, context_usage: float) -> ET.Element: + """ + Generate XML context element with current system metrics. + Clears usage samples after generating averages. + + Args: + context_usage: Current context size usage (0-1) + + Returns: + ET.Element: Context element with system metrics + """ + # Create context element + context = ET.Element("context") + + # Add timestamp + context.set("time", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())) + + # Add usage metrics + with self._metrics_lock: + # CPU average (or 0 if no samples) + cpu_avg = round(sum(self._cpu_samples) / len(self._cpu_samples)) if self._cpu_samples else 0 + context.set("cpu", str(cpu_avg)) + self._cpu_samples.clear() + + # GPU average (or 0 if no samples) + gpu_avg = round(sum(self._gpu_samples) / len(self._gpu_samples)) if self._gpu_samples else 0 + context.set("gpu", str(gpu_avg)) + self._gpu_samples.clear() + + # Memory usage in bytes + memory = psutil.virtual_memory() + context.set("memory_used", str(memory.used)) + context.set("memory_total", str(memory.total)) + + # Disk usage in bytes (root partition) + disk = psutil.disk_usage('/') + context.set("disk_used", str(disk.used)) + context.set("disk_total", str(disk.total)) + + # Context usage (0-100) + context.set("context", str(round(context_usage * 100))) + + # Standard input buffer size + context.set("stdin", "0") # Set by agent + + return context + + def stop(self): + """Stop the monitoring thread and clean up.""" + self._stop_event.set() + self._monitor_thread.join() \ No newline at end of file diff --git a/test/llm_engine_test.py b/test/llm_engine_test.py deleted file mode 100644 index a11b9c5..0000000 --- a/test/llm_engine_test.py +++ /dev/null @@ -1,25 +0,0 @@ -import unittest - -from itertools import tee - -from . import test_data - -from sia.llm_engine import LlmEngine - -class LlmEngineTest(unittest.TestCase): - def setUp(self): - self.model_path = "/root/model" - - def test_initialization(self): - llm_engine = LlmEngine(self.model_path) - self.assertIsInstance(llm_engine, LlmEngine) - - def test_infer(self): - main_context = "This is a test" - llm_engine = LlmEngine(self.model_path) - tokens = llm_engine.infer(test_data.echo_system_prompt, main_context) - print_tokens, result_tokens = tee(tokens) - for token in print_tokens: - print(token, end="", flush=True) - result = ''.join(result_tokens) - self.assertEqual(result, f"{main_context}{main_context}") \ No newline at end of file diff --git a/test/response_parser_test.py b/test/response_parser_test.py new file mode 100644 index 0000000..4df8f2b --- /dev/null +++ b/test/response_parser_test.py @@ -0,0 +1,215 @@ +import unittest +from datetime import datetime +import xml.etree.ElementTree as ET + +from sia.command import Command +from sia.delete_command import DeleteCommand +from sia.stop_command import StopCommand +from sia.entry import Entry +from sia.background_entry import BackgroundEntry +from sia.parse_error_entry import ParseErrorEntry +from sia.read_entry import ReadEntry +from sia.reasoning_entry import ReasoningEntry +from sia.repeat_entry import RepeatEntry +from sia.single_shot_entry import SingleShotEntry +from sia.write_entry import WriteEntry +from sia.web_io_buffer import WebIOBuffer +from sia.response_parser import ResponseParser + +class ResponseParserTest(unittest.TestCase): + def setUp(self): + """Set up test cases with parser and IO buffer""" + self.io_buffer = WebIOBuffer() + self.parser = ResponseParser(self.io_buffer) + + def test_delete_command(self): + """Test parsing delete command""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, DeleteCommand) + self.assertEqual(result.id, "test-id-123") + + def test_delete_command_missing_id(self): + """Test parsing delete command without ID returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("missing required 'id'", result.error) + + def test_stop_command(self): + """Test parsing stop command""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, StopCommand) + + def test_background_entry(self): + """Test parsing background entry""" + xml = 'echo test' + result = self.parser.parse(xml) + + self.assertIsInstance(result, BackgroundEntry) + self.assertEqual(result.script, "echo test") + + def test_background_entry_empty_script(self): + """Test parsing background entry with empty script returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("missing script content", result.error) + + def test_repeat_entry(self): + """Test parsing repeat entry""" + xml = 'ls -l' + result = self.parser.parse(xml) + + self.assertIsInstance(result, RepeatEntry) + self.assertEqual(result.script, "ls -l") + + def test_repeat_entry_empty_script(self): + """Test parsing repeat entry with empty script returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("missing script content", result.error) + + def test_single_shot_entry(self): + """Test parsing single shot entry""" + xml = 'echo "one time"' + result = self.parser.parse(xml) + + self.assertIsInstance(result, SingleShotEntry) + self.assertEqual(result.script, 'echo "one time"') + + def test_single_shot_entry_empty_script(self): + """Test parsing single shot entry with empty script returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("missing script content", result.error) + + def test_reasoning_entry(self): + """Test parsing reasoning entry""" + xml = 'test reasoning' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ReasoningEntry) + self.assertEqual(result.content, "test reasoning") + + def test_reasoning_entry_empty_content(self): + """Test parsing reasoning entry with empty content returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("missing content", result.error) + + def test_read_stdin_entry(self): + """Test parsing read stdin entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ReadEntry) + self.assertEqual(result.io_buffer, self.io_buffer) + + def test_write_stdout_entry(self): + """Test parsing write stdout entry""" + xml = 'test output' + result = self.parser.parse(xml) + + self.assertIsInstance(result, WriteEntry) + self.assertEqual(result.content, "test output") + self.assertEqual(result.io_buffer, self.io_buffer) + + def test_write_stdout_entry_empty_content(self): + """Test parsing write stdout entry with empty content returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("missing content", result.error) + + def test_unknown_element(self): + """Test parsing unknown element returns error entry""" + xml = 'test' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("Unknown root element", result.error) + + def test_malformed_xml(self): + """Test parsing malformed XML returns error entry""" + xml = '' + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("Invalid XML", result.error) + + def test_entry_id_generation(self): + """Test that unique IDs are generated for entries""" + xml1 = 'test 1' + xml2 = 'test 2' + + result1 = self.parser.parse(xml1) + result2 = self.parser.parse(xml2) + + self.assertNotEqual(result1.id, result2.id) + + def test_whitespace_handling(self): + """Test handling of whitespace in XML""" + xml = """ + + test content + with multiple lines + + """ + result = self.parser.parse(xml) + + self.assertIsInstance(result, ReasoningEntry) + self.assertTrue(result.content.strip()) + + def test_script_with_special_characters(self): + """Test parsing script with special characters""" + script = 'echo "Special chars: \n\t\r\'\"\\"' + xml = f'{script}' + + result = self.parser.parse(xml) + self.assertIsInstance(result, SingleShotEntry) + self.assertEqual(result.script, script) + + def test_cdata_content(self): + """Test parsing content with CDATA sections""" + xml = """ + & chars]]> + """ + result = self.parser.parse(xml) + self.assertIsInstance(result, ReasoningEntry) + self.assertEqual(result.content, "Test content with & chars") + + def test_empty_element_with_attributes(self): + """Test parsing empty element with attributes""" + xml = '' + result = self.parser.parse(xml) + self.assertIsInstance(result, ReadEntry) + + def test_error_in_content_extraction(self): + """Test handling errors during content extraction""" + xml = '' # Malformed CDATA + result = self.parser.parse(xml) + + self.assertIsInstance(result, ParseErrorEntry) + self.assertEqual(result.content, xml) + self.assertIn("Invalid XML", result.error) \ No newline at end of file diff --git a/test/system_metrics_test.py b/test/system_metrics_test.py new file mode 100644 index 0000000..3bffc6d --- /dev/null +++ b/test/system_metrics_test.py @@ -0,0 +1,186 @@ +import unittest +import time +import xml.etree.ElementTree as ET +import psutil +import multiprocessing +import math +from typing import List, Tuple + +from sia.system_metrics import SystemMetrics + +def cpu_load_process(): + """Helper process that generates CPU load""" + while True: + math.factorial(100000) + +class SystemMetricsTest(unittest.TestCase): + def setUp(self): + """Create metrics instance with faster sampling for tests""" + self.metrics = SystemMetrics(sample_interval=0.01) + + def tearDown(self): + """Ensure metrics monitoring is stopped""" + self.metrics.stop() + + def validate_timestamp(self, timestamp: str): + """Validate timestamp format and reasonableness""" + try: + # Check format + self.assertRegex( + timestamp, + r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$' + ) + + # Check timestamp is recent (within last minute) + time_parts = timestamp[:-1].split('T') # Remove Z + time_str = f"{time_parts[0]} {time_parts[1]}" + timestamp_time = time.strptime(time_str, "%Y-%m-%d %H:%M:%S") + now = time.gmtime() + + # Convert to seconds since epoch for comparison + timestamp_secs = time.mktime(timestamp_time) + now_secs = time.mktime(now) + + self.assertLess(abs(now_secs - timestamp_secs), 60) + + except Exception as e: + self.fail(f"Invalid timestamp format: {timestamp}") + + def validate_memory_metrics(self, used: int, total: int): + """Validate memory metrics are reasonable""" + # Check types + self.assertIsInstance(used, int) + self.assertIsInstance(total, int) + + # Used memory should be positive and less than total + self.assertGreater(used, 0) + self.assertGreater(total, 0) + self.assertLessEqual(used, total) + + # Compare with actual system memory + memory = psutil.virtual_memory() + self.assertEqual(total, memory.total) + # Used memory should be within 20% of current usage + # (allowing for normal fluctuation) + self.assertLess(abs(used - memory.used) / memory.total, 0.2) + + def validate_disk_metrics(self, used: int, total: int): + """Validate disk metrics are reasonable""" + # Check types + self.assertIsInstance(used, int) + self.assertIsInstance(total, int) + + # Used space should be positive and less than total + self.assertGreater(used, 0) + self.assertGreater(total, 0) + self.assertLessEqual(used, total) + + # Compare with actual disk usage + disk = psutil.disk_usage('/') + self.assertEqual(total, disk.total) + # Used space should match within 1% of actual usage + self.assertLess(abs(used - disk.used) / disk.total, 0.01) + + def validate_usage_values(self, values: List[Tuple[str, int]]): + """Validate usage percentage values are in valid range""" + for name, value in values: + self.assertIsInstance(value, int) + self.assertGreaterEqual( + value, 0, + f"{name} below valid range: {value}" + ) + self.assertLessEqual( + value, 100, + f"{name} above valid range: {value}" + ) + + def test_initial_context(self): + """Test initial context generation""" + context = self.metrics.generate_context(0.5) + + # Verify it's an XML element + self.assertIsInstance(context, ET.Element) + self.assertEqual(context.tag, "context") + + # Validate timestamp + self.validate_timestamp(context.get("time")) + + # Validate memory metrics + self.validate_memory_metrics( + int(context.get("memory_used")), + int(context.get("memory_total")) + ) + + # Validate disk metrics + self.validate_disk_metrics( + int(context.get("disk_used")), + int(context.get("disk_total")) + ) + + # Validate usage percentages + self.validate_usage_values([ + ("CPU", int(context.get("cpu"))), + ("GPU", int(context.get("gpu"))), + ("Context", int(context.get("context"))) + ]) + + # Validate stdin buffer size + self.assertEqual(context.get("stdin"), "0") + + def test_multiple_context_generations(self): + """Test multiple context generations have different timestamps""" + context1 = self.metrics.generate_context(0.5) + time.sleep(1) + context2 = self.metrics.generate_context(0.5) + + self.assertNotEqual( + context1.get("time"), + context2.get("time") + ) + + def test_cpu_usage_detection(self): + """Test CPU usage increases under load""" + # Get initial CPU usage + context1 = self.metrics.generate_context(0.5) + initial_cpu = int(context1.get("cpu")) + + # Generate CPU load in separate process + process = multiprocessing.Process(target=cpu_load_process) + process.start() + + try: + # Wait for samples to accumulate + time.sleep(0.5) + + # Get CPU usage under load + context2 = self.metrics.generate_context(0.5) + load_cpu = int(context2.get("cpu")) + + # CPU usage should increase + self.assertGreater(load_cpu, initial_cpu) + + finally: + process.terminate() + process.join() + + def test_context_usage_reflection(self): + """Test context usage parameter is reflected accurately""" + test_values = [0.0, 0.42, 0.8, 1.0] + + for value in test_values: + context = self.metrics.generate_context(value) + self.assertEqual( + int(context.get("context")), + round(value * 100) + ) + + def test_cleanup(self): + """Test monitoring stops properly""" + self.metrics.stop() + + # Monitor thread should finish + self.assertFalse(self.metrics._monitor_thread.is_alive()) + + # Should still generate valid context after stopping + context = self.metrics.generate_context(0.5) + self.assertIsInstance(context, ET.Element) \ No newline at end of file