Enable 2x faster inference with unsloth

This commit is contained in:
Niels Geens
2025-04-22 11:47:27 +02:00
parent 3e471eced3
commit bf94a0838a

View File

@@ -43,6 +43,9 @@ class QwQLlmEngine(LlmEngine):
gpu_memory_utilization = 0.8, # Reduce if out of memory
tokenizer = self._tokenizer,
)
# enable unsloth optimizations
FastLanguageModel.for_inference(self._model)
# Create inference pipeline
self._pipeline = pipeline(