llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed auto approver and inference continuation

Browse Source
This commit is contained in:
Niels Geens
2025-04-18 11:36:17 +02:00
parent 023f6bba9a
commit c09f0766c1
9 changed files with 304 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
sia/auto_approver.py
View File

@@ -178,7 +178,5 @@ class AutoApprover:
def _response_approval_thread(self) -> None:
if self._stop_event.wait(self._response_timeout):
return
if (self._response_enabled and
self.agent.llms[self._llm_name] == LlmState.IDLE):
response = self.agent.response_buffer.get_text()
if self._response_enabled:
self.agent.approve_response()

7
sia/llm_engine/hf_llm_engine.py
View File

@@ -30,13 +30,14 @@ class HfLlmEngine(LlmEngine):
self._config = AutoConfig.from_pretrained(model, token=api_token)
self._client = InferenceClient(token=api_token)
def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
def infer(self, system_prompt: str, main_context: str, continuation_text: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
"""
Run inference using the system prompt and main context.
Args:
system_prompt: The system prompt string
main_context: The main context string after templating
continuation_text: Part of the response that is already generated
should_stop: Callback that returns True when inference should stop
Returns:
@@ -44,13 +45,15 @@ class HfLlmEngine(LlmEngine):
"""
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": main_context}
{"role": "user", "content": main_context},
{"role": "assistant", "content": continuation_text},
]
stream = self._client.chat_completion(
model=self._model,
messages=messages,
temperature=self._temperature,
add_generation_prompt=False,
stream=True
)

1
sia/llm_engine/local_llm_engine.py
View File

@@ -63,6 +63,7 @@ class LocalLlmEngine(LlmEngine):
Args:
system_prompt: The system prompt string
main_context: The main context string after templating
continuation_text: Part of the response that is already generated
should_stop: Callback that returns True when inference should stop
Returns:

4
sia/llm_engine/openai_llm_engine.py
View File

@@ -34,7 +34,9 @@ class OpenAILlmEngine(LlmEngine):
api_key=api_key,
)
def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
def infer(self, system_prompt: str, main_context: str, continuation_text: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
if continuation_text:
print("OpenAI LLM Engine: continuation_text is not supported")
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": main_context}

1
sia/llm_engine/qwq_llm_engine.py
View File

@@ -70,6 +70,7 @@ class QwQLlmEngine(LlmEngine):
Args:
system_prompt: The system prompt string
main_context: The main context string after templating
continuation_text: Part of the response that is already generated
should_stop: Callback that returns True when inference should stop
Returns:

86
sia/web/api.py
View File

@@ -59,20 +59,25 @@ class Api:
async def _run_inference(self, request: web.Request) -> web.Response:
"""Start inference on specified LLM."""
llm_name = request.match_info["llm"]
try:
llm_name = request.match_info["llm"]
data = await request.json()
text = data.get("response")
context = data.get("context")
self._agent._response_buffer.set_text(text)
self._agent.modify_context(context)
await asyncio.get_event_loop().run_in_executor(None, self._agent.run_inference, llm_name)
return web.Response(status=200)
except (ValueError, RuntimeError) as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _stop_inference(self, request: web.Request) -> web.Response:
"""Stop inference on specified LLM."""
llm_name = request.match_info["llm"]
try:
llm_name = request.match_info["llm"]
self._agent.stop_inference(llm_name)
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _set_response(self, request: web.Request) -> web.Response:
@@ -80,15 +85,10 @@ class Api:
try:
data = await request.json()
text = data.get("response")
if text is None:
return web.Response(status=400, text="Missing response text in request body")
try:
self._agent._response_buffer.set_text(text)
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
except json.JSONDecodeError:
return web.Response(status=400, text="Invalid JSON in request body")
async def _approve_response(self, request: web.Request) -> web.Response:
"""Approve current buffer content"""
@@ -98,35 +98,37 @@ class Api:
self._agent.response_buffer.set_text(response)
self._agent.approve_response()
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _get_response(self, request: web.Request) -> web.Response:
"""Get current buffer content"""
return web.Response(
text=json.dumps({
"response": self._agent.output_buffer.get_text(),
"response": self._agent.response_buffer.get_text(),
}),
content_type="application/json"
)
async def _modify_context(self, request: web.Request) -> web.Response:
"""Modify the current context."""
try:
data = await request.json()
context = data.get("context")
if not context:
return web.Response(status=400, text="Missing context in request body")
self._agent.modify_context(context)
return web.Response(status=200)
except Exception as e:
return web.Response(status=400, text=str(e))
async def _send_input(self, request: web.Request) -> web.Response:
"""Send input to the IO buffer."""
try:
data = await request.json()
input_text = data.get("input")
if not input_text:
return web.Response(status=400, text="Missing input in request body")
self._io_buffer.append_stdin(input_text)
return web.Response(status=200)
except Exception as e:
return web.Response(status=400, text=str(e))
async def _clear_output(self, request: web.Request) -> web.Response:
"""Clear the stdout buffer."""
@@ -135,14 +137,14 @@ class Api:
async def _get_output(self, request: web.Request) -> web.Response:
"""Get complete output for specified LLM."""
llm_name = request.match_info["llm"]
try:
llm_name = request.match_info["llm"]
output = self._agent.get_output(llm_name)
return web.Response(
text=json.dumps({"output": output}),
content_type="application/json"
)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _get_llms(self, request: web.Request) -> web.Response:
@@ -165,71 +167,68 @@ class Api:
async def _set_auto_approver_config(self, request: web.Request) -> web.Response:
"""Update auto approver configuration."""
data = await request.json()
try:
data = await request.json()
self._auto_approver.set_config(data)
return web.Response(status=200)
except (ValueError, KeyError) as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _set_context_enabled(self, request: web.Request) -> web.Response:
"""Set context auto-approval enabled state."""
try:
data = await request.json()
enabled = data.get("enabled")
if enabled is None:
return web.Response(status=400, text="Missing enabled parameter")
try:
self._auto_approver.context_enabled = enabled
self._auto_approver.context_enabled = enabled or False
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _set_response_enabled(self, request: web.Request) -> web.Response:
"""Set response auto-approval enabled state."""
try:
data = await request.json()
enabled = data.get("enabled")
if enabled is None:
return web.Response(status=400, text="Missing enabled parameter")
try:
self._auto_approver.response_enabled = enabled
self._auto_approver.response_enabled = enabled or False
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _set_context_timeout(self, request: web.Request) -> web.Response:
"""Set context auto-approval timeout."""
try:
data = await request.json()
timeout = data.get("timeout")
if timeout is None:
return web.Response(status=400, text="Missing timeout parameter")
try:
self._auto_approver.context_timeout = float(timeout)
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _set_response_timeout(self, request: web.Request) -> web.Response:
"""Set response auto-approval timeout."""
try:
data = await request.json()
timeout = data.get("timeout")
if timeout is None:
return web.Response(status=400, text="Missing timeout parameter")
try:
self._auto_approver.response_timeout = float(timeout)
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _set_llm_name(self, request: web.Request) -> web.Response:
"""Set LLM name for auto-approval."""
try:
data = await request.json()
name = data.get("name")
if name is None:
return web.Response(status=400, text="Missing name parameter")
try:
self._auto_approver.llm_name = name
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _get_memory(self, request: web.Request) -> web.Response:
@@ -244,29 +243,29 @@ class Api:
)
async def _create_entry(self, request: web.Request) -> web.Response:
"""Create a new entry in working memory."""
data = await request.json()
try:
data = await request.json()
entry = EntryFactory.create_entry(data, self._work_dir, self._io_buffer)
self._working_memory.add_entry(entry)
return web.Response(
text=json.dumps({"id": entry.id}),
content_type="application/json"
)
except (ValueError, TypeError) as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _save_entry(self, request: web.Request) -> web.Response:
"""Update properties of an existing entry."""
try:
entry_id = request.match_info["id"]
data = await request.json()
entry = self._working_memory.get_entry(entry_id)
if not entry:
return web.Response(status=404, text="Entry not found")
try:
EntryFactory.update_entry(entry, data)
entry.notify_change()
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _delete_entry(self, request: web.Request) -> web.Response:
@@ -286,19 +285,20 @@ class Api:
async def _update_entry(self, request: web.Request) -> web.Response:
"""Update an entry's state."""
try:
entry_id = request.match_info["id"]
entry = self._working_memory.get_entry(entry_id)
if not entry:
return web.Response(status=404, text="Entry not found")
try:
entry.update()
entry.notify_change()
return web.Response(status=200)
except ValueError as e:
except Exception as e:
return web.Response(status=400, text=str(e))
async def _load_iteration(self, request: web.Request) -> web.Response:
"""Load entries from iteration XML content into working memory"""
try:
data = await request.json()
content = data.get("content")
if not content:
@@ -310,3 +310,5 @@ class Api:
self._working_memory.add_entry(entry)
return web.Response(status=200)
except Exception as e:
return web.Response(status=400, text=str(e))

8
sia/web_agent.py
View File

@@ -97,10 +97,6 @@ class WebAgent(BaseAgent):
"""Update context and reset all LLM states"""
with self._llm_lock:
self._context = context
self._response_buffer.clear()
for llm_name in self._llms:
self._set_llm_state(llm_name, LlmState.IDLE)
for handler in self._context_change_handlers:
handler(context, generated)
@@ -118,7 +114,6 @@ class WebAgent(BaseAgent):
return self._stop_flags[llm_name]
response_token_iter = llm.infer(self.system_prompt, self.context, self._response_buffer.get_text(), should_stop)
for token in response_token_iter:
print(token, end='', flush=True)
with self._output_lock:
self._response_buffer.append_text(token)
with self._llm_lock:
@@ -132,11 +127,10 @@ class WebAgent(BaseAgent):
def approve_response(self) -> None:
"""Process approved response from specified LLM"""
if self.llms.get(llm_name) != LlmState.IDLE:
return
timestamp = datetime.now(timezone.utc)
self._iteration_logger.log_iteration(timestamp, self._context, self._response_buffer.get_text())
parse_result = self._parser.parse(timestamp, self._response_buffer.get_text())
self._response_buffer.clear()
if isinstance(parse_result, Command):
result = parse_result.execute(self._working_memory)
self._command_result = result

23
web/src/components/App.jsx
View File

@@ -16,7 +16,6 @@ const App = () => {
// Editor content state
const [generatedContext, setGeneratedContext] = useState('');
const [modifiedContext, setModifiedContext] = useState('');
const [contextDirty, setContextDirty] = useState(false);
const [generatedResponse, setGeneratedResponse] = useState('');
const [modifiedResponse, setModifiedResponse] = useState('');
const [input, setInput] = useState('');
@@ -65,7 +64,6 @@ const App = () => {
// Handle context changes
useEffect(() => {
contextWs.addMessageHandler((data) => {
setContextDirty(false);
setModifiedContext(data.context);
if (data.generated) {
setGeneratedContext(data.context);
@@ -127,20 +125,13 @@ const App = () => {
};
const handleInference = () => {
fetch('/api/context', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ context: modifiedContext })
});
fetch('/api/response', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ response: modifiedResponse })
});
fetch(`/api/inference/${activeLlm}`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
response: modifiedResponse,
context: modifiedContext,
})
});
};
@@ -176,10 +167,6 @@ const App = () => {
}
setLlms(resetLlms);
setModifiedContext(context);
// Reset response buffers
setGeneratedResponse('');
setModifiedResponse('');
};
const handleResponseEdit = (response) => {