From 3e471eced35dfac87a49175583b3673af2ed8a4f Mon Sep 17 00:00:00 2001 From: Niels Geens Date: Mon, 21 Apr 2025 10:42:35 +0000 Subject: [PATCH] Use Unsloth for QwQ inference --- scripts/install.sh | 1 + setup.py | 9 +- sia/llm_engine/qwq_llm_engine.ipynb | 218 ++++++++++++++ sia/llm_engine/qwq_llm_engine.py | 35 +-- tools/train/setup.py | 3 +- tools/train/train/qwq.ipynb | 208 +++++++++++++ tools/train/train/qwq_train.ipynb | 444 ---------------------------- 7 files changed, 447 insertions(+), 471 deletions(-) create mode 100644 sia/llm_engine/qwq_llm_engine.ipynb create mode 100644 tools/train/train/qwq.ipynb delete mode 100644 tools/train/train/qwq_train.ipynb diff --git a/scripts/install.sh b/scripts/install.sh index 1e75ba7..3bb8059 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -17,4 +17,5 @@ if [ -z "${SIA_INSTALL_NO_CORE}" ]; then echo "Installing SIA core..." python3 -m venv /root/venvs/sia /root/venvs/sia/bin/pip install -e /root/sia + /root/venvs/sia/bin/ipython kernel install --name=sia fi \ No newline at end of file diff --git a/setup.py b/setup.py index a35bf98..0fb6ac4 100644 --- a/setup.py +++ b/setup.py @@ -10,12 +10,13 @@ setup( ], }, install_requires=[ - 'torch>=2.0.0', 'accelerate>=0.26.0', 'aiohttp>=3.8.0', - 'bitsandbytes>=0.41.0', + 'bitsandbytes>=0.45', 'dotenv-python>=0.0.1', 'huggingface_hub>=0.16.0', + 'ipykernel>=6.0.0', + 'ipywidgets>=8.0.0', 'lxml>=4.9.0', 'mistral-common>=1.0.0', 'mistralai>=0.0.7', @@ -23,7 +24,11 @@ setup( 'psutil>=5.9.0', 'python-dotenv>=1.0.0', 'tiktoken>=0.4.0', + 'torch>=2.0.0', 'transformers>=4.30.0', + 'trl>=0.7.8', + 'unsloth>=2025.3', + 'vllm==0.8.2', 'xml_schema_validator @ file:///root/sia/lib/xml_schema_validator', ], python_requires='>=3.10', diff --git a/sia/llm_engine/qwq_llm_engine.ipynb b/sia/llm_engine/qwq_llm_engine.ipynb new file mode 100644 index 0000000..9a29282 --- /dev/null +++ b/sia/llm_engine/qwq_llm_engine.ipynb @@ -0,0 +1,218 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n", + "from unsloth import FastLanguageModel" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "from threading import Thread\n", + "from transformers import AutoTokenizer, TextIteratorStreamer, pipeline\n", + "from xml_schema_validator import XmlLogitsProcessor" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "temperature = 0.6\n", + "model_path = \"/root/models/notebook\"\n", + "xml_schema_text = Path(\"/root/sia/action_schema.xsd\").read_text()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load tokenizer\n", + "tokenizer = AutoTokenizer.from_pretrained(\n", + " model_path,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load model\n", + "model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n", + " model_path,\n", + " load_in_4bit = True, # False for LoRA 16bit\n", + " fast_inference = True, # Enable vLLM fast inference\n", + " gpu_memory_utilization = 0.8, # Reduce if out of memory\n", + " tokenizer = tokenizer,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create inference pipeline with memory-efficient settings\n", + "pipeline = pipeline(\n", + " \"text-generation\",\n", + " model=model,\n", + " tokenizer=tokenizer,\n", + " return_full_text=False,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "logits_processor = XmlLogitsProcessor(tokenizer, xml_schema_text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages = [\n", + " {\"role\": \"system\", \"content\": \"Always respond in a xml block.\"},\n", + " {\"role\": \"user\", \"content\": \"Hi, how are you?\"},\n", + " {\"role\": \"assistant\", \"content\": \"\"},\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "text = tokenizer.apply_chat_template(\n", + " messages,\n", + " tokenize=False,\n", + " add_generation_prompt=False,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "streamer = TextIteratorStreamer(\n", + " tokenizer,\n", + " skip_prompt=True,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generation_kwargs = {\n", + " \"text_inputs\": text,\n", + " \"do_sample\": True,\n", + " \"temperature\": temperature,\n", + " \"streamer\": streamer,\n", + " \"use_cache\": True,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generation_kwargs[\"logits_processor\"] = [logits_processor.copy()]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "generation_thread = Thread(\n", + " target=pipeline,\n", + " kwargs=generation_kwargs\n", + ")\n", + "\n", + "generation_thread.start()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "for text in streamer:\n", + " print(text, end=\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "ename": "", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[1;31mThe Kernel crashed while executing code in the current cell or a previous cell. \n", + "\u001b[1;31mPlease review the code in the cell(s) to identify a possible cause of the failure. \n", + "\u001b[1;31mClick here for more info. \n", + "\u001b[1;31mView Jupyter log for further details." + ] + } + ], + "source": [ + "\n", + "generation_thread.join()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "sia", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/sia/llm_engine/qwq_llm_engine.py b/sia/llm_engine/qwq_llm_engine.py index 1c0d0bb..f8c0f00 100644 --- a/sia/llm_engine/qwq_llm_engine.py +++ b/sia/llm_engine/qwq_llm_engine.py @@ -1,6 +1,9 @@ +# Unsloth should be imported before transformers to ensure all optimizations are applied. +from unsloth import FastLanguageModel + from pathlib import Path from threading import Thread -from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer, pipeline, BitsAndBytesConfig +from transformers import AutoTokenizer, TextIteratorStreamer, pipeline from typing import Callable, Iterator, Optional from xml_schema_validator import XmlLogitsProcessor import os @@ -26,40 +29,27 @@ class QwQLlmEngine(LlmEngine): xml_schema_text: Optional XML schema to validate against """ self._temperature = temperature - - # Configure 4-bit quantization for massive memory savings - quantization_config = BitsAndBytesConfig( - load_in_4bit=True, - bnb_4bit_compute_dtype=torch.bfloat16, - bnb_4bit_use_double_quant=True, - bnb_4bit_quant_type="nf4" - ) - # Load tokenizer first - this uses minimal memory + # Load tokenizer self._tokenizer = AutoTokenizer.from_pretrained( model_path, - padding_side="left", - trust_remote_code=True, ) - # Load model with 4-bit quantization - self._model = AutoModelForCausalLM.from_pretrained( + # Load model + self._model, _returned_tokenizer = FastLanguageModel.from_pretrained( model_path, - device_map="auto", - quantization_config=quantization_config, - torch_dtype=torch.bfloat16, - low_cpu_mem_usage=True, - trust_remote_code=True, + load_in_4bit = True, # False for LoRA 16bit + fast_inference = True, # Enable vLLM fast inference + gpu_memory_utilization = 0.8, # Reduce if out of memory + tokenizer = self._tokenizer, ) - # Create inference pipeline with memory-efficient settings + # Create inference pipeline self._pipeline = pipeline( "text-generation", model=self._model, tokenizer=self._tokenizer, return_full_text=False, - device_map="auto", - torch_dtype=torch.bfloat16, ) if xml_schema_text: @@ -102,7 +92,6 @@ class QwQLlmEngine(LlmEngine): "text_inputs": text, "do_sample": True, "temperature": self._temperature, - "max_new_tokens": self.token_limit(), "streamer": streamer, "use_cache": True, } diff --git a/tools/train/setup.py b/tools/train/setup.py index 4f1c9de..6406e95 100644 --- a/tools/train/setup.py +++ b/tools/train/setup.py @@ -9,7 +9,7 @@ setup( ], install_requires=[ - 'accelerate>=0.25.0', + 'accelerate>=0.26.0', 'bitsandbytes>=0.45.0', 'black>=22.0.0', 'datasets>=2.14.6', @@ -18,7 +18,6 @@ setup( 'ipykernel>=6.0.0', 'ipywidgets>=8.0.0', 'peft>=0.8.0', - 'peft>=0.8.0', 'pytest-cov>=4.0.0', 'pytest>=7.0.0', 'pyyaml>=6.0', diff --git a/tools/train/train/qwq.ipynb b/tools/train/train/qwq.ipynb new file mode 100644 index 0000000..513604d --- /dev/null +++ b/tools/train/train/qwq.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n", + "from unsloth import FastLanguageModel, is_bfloat16_supported" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from dataclasses import dataclass\n", + "from pathlib import Path\n", + "from transformers import AutoTokenizer, TrainingArguments\n", + "from trl import SFTTrainer, DataCollatorForCompletionOnlyLM\n", + "from typing import Optional, List\n", + "import argparse\n", + "import json\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from train import qwq" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "args = qwq.Args([\"--output-dir\", \"/root/models/notebook\"])" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset = qwq.Dataset(args.config_path)\n", + "dataset.validate()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "max_seq_length = 2048 # Can increase for longer reasoning traces\n", + "lora_rank = 64 # Larger rank = smarter, but slower" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "with open('/root/sia/qwq_tokenizer_config.json', 'r') as f:\n", + " tokenizer_config = json.load(f)\n", + "\n", + "tokenizer = AutoTokenizer.from_pretrained(\n", + " args.base_model,\n", + " **tokenizer_config,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n", + " model_name = args.base_model,\n", + " max_seq_length = max_seq_length,\n", + " load_in_4bit = True, # False for LoRA 16bit\n", + " fast_inference = True, # Enable vLLM fast inference\n", + " max_lora_rank = lora_rank,\n", + " gpu_memory_utilization = 0.5, # Reduce if out of memory\n", + " tokenizer = tokenizer,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model = FastLanguageModel.get_peft_model(\n", + " model,\n", + " r = lora_rank, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n", + " target_modules = [\n", + " \"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", + " \"gate_proj\", \"up_proj\", \"down_proj\",\n", + " ], # Remove QKVO if out of memory\n", + " lora_alpha = lora_rank,\n", + " lora_dropout = 0, # Supports any, but = 0 is optimized\n", + " bias = \"none\", # Supports any, but = \"none\" is optimized\n", + " use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n", + " random_state = 3407,\n", + " use_rslora = False, # We support rank stabilized LoRA\n", + " loftq_config = None, # And LoftQ\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "training_args = TrainingArguments(\n", + " output_dir=str(args.output_dir),\n", + " num_train_epochs=3,\n", + " per_device_train_batch_size=1,\n", + " gradient_accumulation_steps=16,\n", + " gradient_checkpointing=True,\n", + " learning_rate=2e-5,\n", + " lr_scheduler_type=\"cosine\",\n", + " warmup_ratio=0.05,\n", + " weight_decay=0.01,\n", + " fp16=not is_bfloat16_supported(),\n", + " bf16=is_bfloat16_supported(),\n", + " logging_steps=10,\n", + " save_steps=200,\n", + " save_total_limit=3,\n", + " report_to=\"none\",\n", + " optim=\"adamw_8bit\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trainer = SFTTrainer(\n", + " model=model,\n", + " tokenizer=tokenizer,\n", + " args=training_args,\n", + " train_dataset=dataset.to_transformers_dataset(tokenizer),\n", + " dataset_text_field=\"messages\",\n", + " max_seq_length=max_seq_length,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "model.save_pretrained_merged(\n", + " str(args.output_dir), \n", + " tokenizer=tokenizer,\n", + " #save_method=\"merged_4bit_forced\"\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/tools/train/train/qwq_train.ipynb b/tools/train/train/qwq_train.ipynb deleted file mode 100644 index 085bc87..0000000 --- a/tools/train/train/qwq_train.ipynb +++ /dev/null @@ -1,444 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", - "🦥 Unsloth Zoo will now patch everything to make training faster!\n", - "INFO 03-30 08:37:14 [__init__.py:239] Automatically detected platform cuda.\n" - ] - } - ], - "source": [ - "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n", - "from unsloth import FastLanguageModel, is_bfloat16_supported" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from dataclasses import dataclass\n", - "from pathlib import Path\n", - "from transformers import TrainingArguments\n", - "from trl import SFTTrainer, DataCollatorForCompletionOnlyLM\n", - "from typing import Optional, List\n", - "import argparse\n", - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [], - "source": [ - "from train import qwq" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "args = qwq.Args([\"--output-dir\", \"/root/models/notebook\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Validating 20 XML files...\n", - "file: /root/sia/training/clean_start/iteration_20250116_134549_655.xml\n", - "file: /root/sia/training/clean_start/iteration_20250116_134555_680.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141241_092.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141252_317.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141302_940.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141329_886.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141343_416.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141357_412.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141410_965.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141428_204.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141441_443.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141447_231.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141454_509.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141458_495.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141503_889.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141516_718.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141533_231.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141603_549.xml\n", - "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141633_083.xml\n", - "file: /root/sia/training/list_entries_to_delete/iteration_20250116_141227_271.xml\n", - "Validation complete. Found 20 valid files.\n" - ] - } - ], - "source": [ - "dataset = qwq.Dataset(args.config_path)\n", - "dataset.validate()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "max_seq_length = 2048 # Can increase for longer reasoning traces\n", - "lora_rank = 64 # Larger rank = smarter, but slower" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "==((====))== Unsloth 2025.3.19: Fast Qwen2 patching. Transformers: 4.50.3. vLLM: 0.8.2.\n", - " \\\\ /| NVIDIA RTX 6000 Ada Generation. Num GPUs = 1. Max memory: 47.5 GB. Platform: Linux.\n", - "O^O/ \\_/ \\ Torch: 2.6.0+cu124. CUDA: 8.9. CUDA Toolkit: 12.4. Triton: 3.2.0\n", - "\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.29.post2. FA2 = False]\n", - " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", - "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n", - "Unsloth: vLLM loading unsloth/qwq-32b-unsloth-bnb-4bit with actual GPU utilization = 49.51%\n", - "Unsloth: Your GPU has CUDA compute capability 8.9 with VRAM = 47.5 GB.\n", - "Unsloth: Using conservativeness = 1.0. Chunked prefill tokens = 2048. Num Sequences = 128.\n", - "Unsloth: vLLM's KV Cache can use up to 1.46 GB. Also swap space = 6 GB.\n", - "INFO 03-30 08:37:24 [config.py:585] This model supports multiple tasks: {'score', 'classify', 'reward', 'generate', 'embed'}. Defaulting to 'generate'.\n", - "WARNING 03-30 08:37:24 [arg_utils.py:1854] --quantization bitsandbytes is not supported by the V1 Engine. Falling back to V0. \n", - "Unsloth: vLLM Bitsandbytes config using kwargs = {'load_in_8bit': False, 'load_in_4bit': True, 'bnb_4bit_compute_dtype': 'bfloat16', 'bnb_4bit_quant_storage': 'uint8', 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': True, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'llm_int8_skip_modules': ['lm_head', 'multi_modal_projector', 'merger', 'modality_projection', 'model.layers.4.mlp', 'model.layers.0.mlp', 'model.layers.60.mlp', 'model.layers.62.mlp', 'model.layers.5.mlp', 'model.layers.43.self_attn'], 'llm_int8_threshold': 6.0}\n", - "INFO 03-30 08:37:24 [llm_engine.py:241] Initializing a V0 LLM engine (v0.8.2) with config: model='unsloth/qwq-32b-unsloth-bnb-4bit', speculative_config=None, tokenizer='unsloth/qwq-32b-unsloth-bnb-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=bitsandbytes, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=bitsandbytes, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda:0, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=unsloth/qwq-32b-unsloth-bnb-4bit, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={\"level\":0,\"splitting_ops\":[],\"compile_sizes\":[],\"cudagraph_capture_sizes\":[128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],\"max_capture_size\":128}, use_cached_outputs=False, \n", - "INFO 03-30 08:37:24 [cuda.py:291] Using Flash Attention backend.\n", - "INFO 03-30 08:37:25 [parallel_state.py:954] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0\n", - "INFO 03-30 08:37:25 [model_runner.py:1110] Starting to load model unsloth/qwq-32b-unsloth-bnb-4bit...\n", - "INFO 03-30 08:37:25 [loader.py:1155] Loading weights with BitsAndBytes quantization. May take a while ...\n", - "INFO 03-30 08:37:25 [weight_utils.py:265] Using model weights format ['*.safetensors']\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "198d11b20e02412cb2833c5042d2af97", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00 0 ! Suggested 8, 16, 32, 64, 128\n", - " target_modules = [\n", - " \"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n", - " \"gate_proj\", \"up_proj\", \"down_proj\",\n", - " ], # Remove QKVO if out of memory\n", - " lora_alpha = lora_rank,\n", - " lora_dropout = 0, # Supports any, but = 0 is optimized\n", - " bias = \"none\", # Supports any, but = \"none\" is optimized\n", - " use_gradient_checkpointing = \"unsloth\", # True or \"unsloth\" for very long context\n", - " random_state = 3407,\n", - " use_rslora = False, # We support rank stabilized LoRA\n", - " loftq_config = None, # And LoftQ\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'messages': [{'role': 'system',\n", - " 'content': 'You are SIA, the Self Improving Agent.\\nYour goal is to autonomously complete complex tasks by writing and executing scripts.\\nYou can solve any problem.\\n\\nEach iteration, the context is updated with the result of your previous actions.\\nYou modify the context by issuing a command using XML.\\nParameters and scripts may be long and complex.\\nUse correct XML escaping or CDATA sections.\\nIt is very important that you always respond with one action adhering to the XML schema!\\nDo not respond with anything else after the first action.\\n\\nThe next iteration starts when all scripts have finished.\\nThese are repeat scripts from the previous iterations and possibly one new single-shot script.\\nAvoid blocking scripts so you can iterate quickly.\\n\\n# Context\\n\\nThe context has a limited length.\\nThe `context_usage` attribute of the main context element indicates how much of the context is used in %.\\nThis should never reach 100%!\\nUse the delete action to remove unnecessary items from the context.\\nBut keep interesting information.\\nYou can\\'t learn from your mistakes if you delete them before fixing.\\n\\n# Linux Environment\\n\\nYou have access to the Linux environment that runs the SAI process.\\nIn this environment you can run scripts by issuing the right actions.\\nScripts and their output appear in the context.\\nYou can use a script for starting a detached process that runs in the background.\\nAll processes can be managed by the usual Linux tools.\\nThe scripts defined in the script actions all run in a `bash` shell.\\nYou are logged in as root.\\n\\n# File system\\n\\nThe file system helps you structure your thoughts.\\nBecause of the limited context window you can\\'t remember everything you\\'ve done and learned.\\nWriting and updating files will help you in:\\n- remembering tasks\\n- planning solution strategies\\n- keeping track of progress\\n- managing overview of large projects\\n- using tools you\\'ve created\\n\\nIt is important to bring a lot of structure to the files and directories.\\nThis will help you find the right info when needed.\\nWhen solving a problem, make sure to load the relevant info in context before planning.\\nYou can load a single file with a `cat` command executed in a `single` action.\\n`head`, `tail`, `grep`, `find`, `tree`, ... all have their uses.\\n\\nFor code source files it may be interesting to add line numbers.\\nMore advanced scripts can be used, for instance to extract documentation from source files.\\nThis helps you to know how to use a file without loading all the code in context too.\\n\\nIf it isn\\'t clear what you should do next, check the filesystem for notes that may guide you!\\n\\n# Iterative Problem Solving\\n\\nTake small steps and verify your work.\\nCreate unit tests for all your work so you can do regression tests after each step.\\n\\nKeep notes of when you started on a subtask and which solutions you tried.\\nThis way you avoid repeating yourself and decide when to look for an alternative approach to a problem.\\n\\nVersion control tools help remember steps taken, solutions tried and files modified.\\nMake extensive use of `git`!\\n\\nYour most important tool is the reasoning action.\\nYou should reason about everything you\\'ll do before issuing a command in the next iteration!\\nInspect your previous actions in detail.\\nIf something didn\\'t work, try to understand why and don\\'t repeat the same mistake.\\nDon\\'t delete mistakes until you understand them.\\n\\nIf you notice parse_error entries in the context you have made a mistake.\\nReason about the error before trying again.\\n\\n# User interaction\\n\\nYou are always working for a user.\\nGet to know them and make notes about what you learn from them.\\nBe a helpful assistant to the user.\\nOpen the relevant user notes when you interact with them.\\n\\nThe main way to communicate is using standard io.\\nThe user may want you to set up alternative communication methods.\\nUse scripts and background processes to do so.\\n\\nThe user may take some time to respond or may forget to respond.\\nKeep detailed notes of your interactions and your expectations regarding time!\\nAvoid overflowing the user with many messages.\\n\\n\\n\\n\\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n\\n'},\n", - " {'role': 'user',\n", - " 'content': '\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nAt node \"evaluate_test_results\". Entry 45f3d2 shows failed test: \"Error: Connection timeout\". \\nWill need to check system logs soon (noted in /tasks/reminders.txt, check at 14:00).\\nFirst focusing on this error.\\n\\n```\\n\\n## Reasons to Switch Procedures\\n\\nCommon triggers:\\n - Data available on stdin\\n - Time matching scheduled task\\n - Error conditions in script output\\n - Resource constraints detected\\n - User input needed]]>\\n \\n \\n \\n \\n \\n \\n Also load the last 10 messages from /user/conversation_history/ ]\\n PrepareForDraft{Have everything needed for drafting a message?}\\n DraftMessage[Draft message in reasoning entry]\\n ReadInput[Read input from standard input]\\n ReasonCleanContext[List id\\'s of entries that are no longer needed
Explain for each entry why it is no longer needed]\\n DeleteEntries[Remove the entries that are no longer needed
End by deleting the ReasonCleanContext entry]\\n AddHistoryUser[Add the message to the /user/conversation_history/ directory
The filename is the id of the stdin entry with .user extension]\\n LoadTask[Look for the task in the /tasks directory and load relevant files]\\n LoadUserDetails[Look in the /user directory for relevant files]\\n EstimateScript[Draft the script in a reasoning block and estimate its runtime and output length]\\n ScriptAcceptable{Does the draft script make sense and are the estimations short enough to not hinder the conversation?}\\n RunScript[Run the script, make sure to set appropriate timeout and output limits]\\n ReviewDraft{Is the message well structured and free of logical errors?}\\n SendMessage[Send the message using standard output]\\n AddHistoryAgent[Add the message to the /user/conversation_history/ directory
The filename is the id of the stdout entry with .agent extension]\\n ReasonResponse[Is the conversation ongoing?
How long is the user expected to take to respond?]\\n NeedAwaitResponse{Is it likely to get a response within a minute?}\\n BusyWait[Wait 1 second for the first busy wait, double the time each iteration until a response is received
Make sure to set the timout]\\n\\n End([Clean the context])\\n\\n Start --> LoadUserBasic\\n LoadUserBasic --> PrepareForDraft\\n\\n PrepareForDraft -->|Got all needed info| DraftMessage\\n PrepareForDraft -->|Getting the required info would slow the conversation| DraftMessage\\n PrepareForDraft -->|Input available on stdin| ReadInput\\n PrepareForDraft -->|Context usage more than 50%| ReasonCleanContext\\n PrepareForDraft -->|Task mentioned but not loaded| LoadTask\\n PrepareForDraft -->|Personal or social info mentioned but not loaded| LoadUserDetails\\n PrepareForDraft -->|Calculations, system info or other numerical values that can be scripted are mentioned| EstimateScript\\n\\n ReasonCleanContext --> DeleteEntries\\n DeleteEntries --> PrepareForDraft\\n\\n ReadInput --> AddHistoryUser\\n AddHistoryUser --> PrepareForDraft\\n\\n LoadTask --> PrepareForDraft\\n LoadUserDetails --> PrepareForDraft\\n\\n EstimateScript --> ScriptAcceptable\\n ScriptAcceptable -->|Acceptable| RunScript\\n ScriptAcceptable -->|Not acceptable| PrepareForDraft\\n RunScript --> PrepareForDraft\\n \\n DraftMessage --> ReviewDraft{Is this really what I want to say?}\\n ReviewDraft -->|Rewrite better| DraftMessage\\n ReviewDraft -->|Good message| SendMessage\\n \\n SendMessage --> AddHistoryAgent\\n AddHistoryAgent --> ReasonResponse\\n ReasonResponse --> NeedAwaitResponse\\n \\n NeedAwaitResponse -->|Quick response is unlikely| End\\n NeedAwaitResponse -->|Input available on stdin| PrepareForDraft\\n NeedAwaitResponse -->|Quick response is likely| BusyWait\\n BusyWait --> NeedAwaitResponse\\n```]]>\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n'},\n", - " {'role': 'assistant', 'content': ''}]}" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "dataset[5]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "training_args = TrainingArguments(\n", - " output_dir=str(args.output_dir),\n", - " num_train_epochs=3,\n", - " per_device_train_batch_size=1,\n", - " gradient_accumulation_steps=16,\n", - " gradient_checkpointing=True,\n", - " learning_rate=2e-5,\n", - " lr_scheduler_type=\"cosine\",\n", - " warmup_ratio=0.05,\n", - " weight_decay=0.01,\n", - " fp16=not is_bfloat16_supported(),\n", - " bf16=is_bfloat16_supported(),\n", - " logging_steps=10,\n", - " save_steps=200,\n", - " save_total_limit=3,\n", - " report_to=\"none\",\n", - " optim=\"adamw_8bit\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "num_proc must be <= 20. Reducing num_proc to 20 for dataset of size 20.\n" - ] - } - ], - "source": [ - "trainer = SFTTrainer(\n", - " model=model,\n", - " tokenizer=tokenizer,\n", - " args=training_args,\n", - " train_dataset=dataset.to_transformers_dataset(tokenizer),\n", - " dataset_text_field=\"messages\",\n", - " max_seq_length=max_seq_length,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", - " \\\\ /| Num examples = 20 | Num Epochs = 3 | Total steps = 3\n", - "O^O/ \\_/ \\ Batch size per device = 1 | Gradient accumulation steps = 16\n", - "\\ / Data Parallel GPUs = 1 | Total batch size (1 x 16 x 1) = 16\n", - " \"-____-\" Trainable parameters = 536,870,912/32,000,000,000 (1.68% trained)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unsloth: Will smartly offload gradients to save VRAM!\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "
\n", - " \n", - " \n", - " [3/3 01:45, Epoch 1/3]\n", - "
\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining Loss

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "TrainOutput(global_step=3, training_loss=1.7514243125915527, metrics={'train_runtime': 187.9691, 'train_samples_per_second': 0.319, 'train_steps_per_second': 0.016, 'total_flos': 7927521441988608.0, 'train_loss': 1.7514243125915527})" - ] - }, - "execution_count": 12, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "trainer.train()" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unsloth: Merging 4bit and LoRA weights to 16bit...\n", - "Unsloth: Will use up to 312.94 out of 503.54 RAM for saving.\n", - "Unsloth: Saving model... This might take 5 minutes ...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - " 12%|█▎ | 8/64 [00:00<00:02, 22.74it/s]\n", - "We will save to Disk and not RAM now.\n", - "100%|██████████| 64/64 [00:43<00:00, 1.48it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Unsloth: Saving tokenizer... Done.\n", - "Done.\n" - ] - } - ], - "source": [ - "model.save_pretrained_merged(\n", - " str(args.output_dir), \n", - " tokenizer=tokenizer,\n", - " #save_method=\"merged_4bit_forced\"\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "train", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}