llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Shared response buffer backend

Browse Source
This commit is contained in:
Niels Geens
2025-04-17 18:32:23 +02:00
parent 9477575421
commit 34fb5d814f
12 changed files with 941 additions and 869 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
sia/llm_engine/qwq_llm_engine.py
View File

@@ -63,7 +63,7 @@ class QwQLlmEngine(LlmEngine):
self._logits_processor = None
def infer(self, system_prompt: str, main_context: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
def infer(self, system_prompt: str, main_context: str, continuation_text: str, should_stop: Callable[[], bool] = lambda: False) -> Iterator[str]:
"""
Run inference using the system prompt and main context.
@@ -77,13 +77,14 @@ class QwQLlmEngine(LlmEngine):
"""
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": main_context}
{"role": "user", "content": main_context},
{"role": "assistant", "content": continuation_text},
]
text = self._tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
add_generation_prompt=False,
)
streamer = TextIteratorStreamer(