Implement 4-bit quant for train script and inference

sia/llm_engine/qwq_llm_engine.py

@@ -6,8 +6,6 @@ from threading import Thread
 from transformers import AutoTokenizer, TextIteratorStreamer, pipeline
 from typing import Callable, Iterator, Optional
 from xml_schema_validator import XmlLogitsProcessor
-import os
-import torch
 
 from . import LlmEngine
 from .. import util
@@ -38,10 +36,7 @@ class QwQLlmEngine(LlmEngine):
         # Load model
         self._model, _returned_tokenizer = FastLanguageModel.from_pretrained(
             model_path,
-            load_in_4bit = True, # False for LoRA 16bit
-            fast_inference = True, # Enable vLLM fast inference
-            gpu_memory_utilization = 0.8, # Reduce if out of memory
-            tokenizer = self._tokenizer,
+            gpu_memory_utilization = 0.5, # Reduce if out of memory
         )
 
         # enable unsloth optimizations