Enable 2x faster inference with unsloth

This commit is contained in:
Niels Geens
2025-04-22 11:47:27 +02:00
parent 3e471eced3
commit bf94a0838a

View File

@@ -44,6 +44,9 @@ class QwQLlmEngine(LlmEngine):
tokenizer = self._tokenizer, tokenizer = self._tokenizer,
) )
# enable unsloth optimizations
FastLanguageModel.for_inference(self._model)
# Create inference pipeline # Create inference pipeline
self._pipeline = pipeline( self._pipeline = pipeline(
"text-generation", "text-generation",