Enable 2x faster inference with unsloth

This commit is contained in:
Niels Geens
2025-04-22 11:47:27 +02:00
parent 3e471eced3
commit bf94a0838a

View File

@@ -44,6 +44,9 @@ class QwQLlmEngine(LlmEngine):
tokenizer = self._tokenizer,
)
# enable unsloth optimizations
FastLanguageModel.for_inference(self._model)
# Create inference pipeline
self._pipeline = pipeline(
"text-generation",