From bf94a0838aeeeebe18340cd89f9f3b0f11bc8e5e Mon Sep 17 00:00:00 2001
From: Niels Geens <niels.geens@materialise.be>
Date: Tue, 22 Apr 2025 11:47:27 +0200
Subject: [PATCH] Enable 2x faster inference with unsloth

---
 sia/llm_engine/qwq_llm_engine.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sia/llm_engine/qwq_llm_engine.py b/sia/llm_engine/qwq_llm_engine.py
index f8c0f00..8283681 100644
--- a/sia/llm_engine/qwq_llm_engine.py
+++ b/sia/llm_engine/qwq_llm_engine.py
@@ -43,6 +43,9 @@ class QwQLlmEngine(LlmEngine):
             gpu_memory_utilization = 0.8, # Reduce if out of memory
             tokenizer = self._tokenizer,
         )
+
+        # enable unsloth optimizations
+        FastLanguageModel.for_inference(self._model)
         
         # Create inference pipeline
         self._pipeline = pipeline(