Updated notebooks to support 4-bit quantization

sia/llm_engine/qwq_llm_engine.ipynb

@@ -2,9 +2,20 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
+      "Unsloth: Failed to patch Gemma3ForConditionalGeneration.\n",
+      "🦥 Unsloth Zoo will now patch everything to make training faster!\n",
+      "INFO 04-23 16:36:10 [__init__.py:239] Automatically detected platform cuda.\n"
+     ]
+    }
+   ],
    "source": [
     "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n",
     "from unsloth import FastLanguageModel"
@@ -12,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -24,18 +35,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
     "temperature = 0.6\n",
-    "model_path = \"/root/models/notebook\"\n",
+    "model_path = \"/root/models/notebook_merged_4bit\"\n",
     "xml_schema_text = Path(\"/root/sia/action_schema.xsd\").read_text()"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -47,24 +58,74 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "==((====))==  Unsloth 2025.3.19: Fast Qwen2 patching. Transformers: 4.51.3. vLLM: 0.8.2.\n",
+      "   \\\\   /|    NVIDIA RTX 6000 Ada Generation. Num GPUs = 1. Max memory: 47.5 GB. Platform: Linux.\n",
+      "O^O/ \\_/ \\    Torch: 2.6.0+cu124. CUDA: 8.9. CUDA Toolkit: 12.4. Triton: 3.2.0\n",
+      "\\        /    Bfloat16 = TRUE. FA [Xformers = 0.0.29.post2. FA2 = False]\n",
+      " \"-____-\"     Free license: http://github.com/unslothai/unsloth\n",
+      "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Sliding Window Attention is enabled but not implemented for `eager`; unexpected results may be encountered.\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "validate_env\n",
+      "device_map sequential\n",
+      "validate_env\n",
+      "device_map OrderedDict([('', 0)])\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "a99acfd42fa645abae77144125a734b7",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "# Load model\n",
     "model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n",
     "    model_path,\n",
-    "    load_in_4bit = True, # False for LoRA 16bit\n",
-    "    fast_inference = True, # Enable vLLM fast inference\n",
     "    gpu_memory_utilization = 0.8, # Reduce if out of memory\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Device set to use cuda:0\n"
+     ]
+    }
+   ],
    "source": [
     "# Create inference pipeline with memory-efficient settings\n",
     "pipeline = pipeline(\n",
@@ -77,7 +138,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -86,7 +147,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -99,7 +160,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -112,7 +173,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -124,7 +185,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -139,7 +200,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -148,7 +209,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -162,9 +223,18 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 14,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<write_stdout>\n",
+      "Hello! I'm just a large language model, so I don't have feelings, but thank you for asking. How can I assist you today?</write_stdout><|im_end|>"
+     ]
+    }
+   ],
    "source": [
     "for text in streamer:\n",
     "    print(text, end=\"\")"
@@ -172,7 +242,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 15,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -185,7 +255,7 @@
   "kernelspec": {
    "display_name": "sia",
    "language": "python",
-   "name": "sia"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {