diff --git a/sia/llm_engine/qwq_llm_engine.py b/sia/llm_engine/qwq_llm_engine.py index f8c0f00..8283681 100644 --- a/sia/llm_engine/qwq_llm_engine.py +++ b/sia/llm_engine/qwq_llm_engine.py @@ -43,6 +43,9 @@ class QwQLlmEngine(LlmEngine): gpu_memory_utilization = 0.8, # Reduce if out of memory tokenizer = self._tokenizer, ) + + # enable unsloth optimizations + FastLanguageModel.for_inference(self._model) # Create inference pipeline self._pipeline = pipeline(