diff --git a/tools/train/train/qwq_infer.ipynb b/tools/train/train/qwq_infer.ipynb index 200ca37..9f3ac61 100644 --- a/tools/train/train/qwq_infer.ipynb +++ b/tools/train/train/qwq_infer.ipynb @@ -2,25 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "execution_count": 1, + "metadata": {}, "outputs": [], "source": [ - "from transformers import AutoModelForCausalLM, AutoTokenizer" + "from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline, TextIteratorStreamer, BitsAndBytesConfig\n", + "from threading import Thread\n", + "import torch" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "execution_count": 2, + "metadata": {}, "outputs": [], "source": [ "model_path = \"/root/models/notebook\"" @@ -28,30 +22,60 @@ }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "execution_count": 3, + "metadata": {}, "outputs": [], "source": [ - "model = AutoModelForCausalLM.from_pretrained(\n", - " model_path,\n", - " return_dict=True,\n", - " torch_dtype=\"auto\",\n", - " device_map=\"auto\"\n", + "quantization_config = BitsAndBytesConfig(\n", + " load_in_4bit=True,\n", + " bnb_4bit_compute_dtype=torch.bfloat16,\n", + " bnb_4bit_quant_type=\"nf4\",\n", + " bnb_4bit_use_double_quant=True\n", ")" ] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "5f35aa01aafa4bc2beb19f437373945e", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading checkpoint shards: 0%| | 0/14 [00:00\n", + "\n", + "The word \"strawberry\" contains three 'r's. Here's the breakdown:\n", + "\n", + "1. **S** \n", + "2. **T** \n", + "3. **R** (first 'r') \n", + "4. **A** \n", + "5. **W** \n", + "6. **B** \n", + "7. **E** \n", + "8. **R** (second 'r') \n", + "9. **R** (third 'r') \n", + "10. **Y** \n", + "\n", + "So, **there are 3 'r's in \"strawberry\"**.<|im_end|>" + ] } - }, - "outputs": [], + ], "source": [ - "\n", "for text in streamer:\n", - " print(text)" + " print(text, end='')" ] } ], "metadata": { + "kernelspec": { + "display_name": "train", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" } }, "nbformat": 4, diff --git a/tools/train/train/qwq_train.ipynb b/tools/train/train/qwq_train.ipynb index f7fa300..ad0156b 100644 --- a/tools/train/train/qwq_train.ipynb +++ b/tools/train/train/qwq_train.ipynb @@ -2,9 +2,19 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n", + "🦥 Unsloth Zoo will now patch everything to make training faster!\n", + "INFO 03-30 08:37:14 [__init__.py:239] Automatically detected platform cuda.\n" + ] + } + ], "source": [ "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n", "from unsloth import FastLanguageModel, is_bfloat16_supported" @@ -12,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -27,7 +37,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -36,7 +46,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -45,9 +55,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Validating 20 XML files...\n", + "file: /root/sia/training/clean_start/iteration_20250116_134549_655.xml\n", + "file: /root/sia/training/clean_start/iteration_20250116_134555_680.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141241_092.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141252_317.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141302_940.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141329_886.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141343_416.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141357_412.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141410_965.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141428_204.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141441_443.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141447_231.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141454_509.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141458_495.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141503_889.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141516_718.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141533_231.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141603_549.xml\n", + "file: /root/sia/training/delete_indicated_entries/iteration_20250116_141633_083.xml\n", + "file: /root/sia/training/list_entries_to_delete/iteration_20250116_141227_271.xml\n", + "Validation complete. Found 20 valid files.\n" + ] + } + ], "source": [ "dataset = qwq.Dataset(args.config_path)\n", "dataset.validate()" @@ -55,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -65,9 +104,100 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "==((====))== Unsloth 2025.3.19: Fast Qwen2 patching. Transformers: 4.50.3. vLLM: 0.8.2.\n", + " \\\\ /| NVIDIA RTX 6000 Ada Generation. Num GPUs = 1. Max memory: 47.5 GB. Platform: Linux.\n", + "O^O/ \\_/ \\ Torch: 2.6.0+cu124. CUDA: 8.9. CUDA Toolkit: 12.4. Triton: 3.2.0\n", + "\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.29.post2. FA2 = False]\n", + " \"-____-\" Free license: http://github.com/unslothai/unsloth\n", + "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n", + "Unsloth: vLLM loading unsloth/qwq-32b-unsloth-bnb-4bit with actual GPU utilization = 49.51%\n", + "Unsloth: Your GPU has CUDA compute capability 8.9 with VRAM = 47.5 GB.\n", + "Unsloth: Using conservativeness = 1.0. Chunked prefill tokens = 2048. Num Sequences = 128.\n", + "Unsloth: vLLM's KV Cache can use up to 1.46 GB. Also swap space = 6 GB.\n", + "INFO 03-30 08:37:24 [config.py:585] This model supports multiple tasks: {'score', 'classify', 'reward', 'generate', 'embed'}. Defaulting to 'generate'.\n", + "WARNING 03-30 08:37:24 [arg_utils.py:1854] --quantization bitsandbytes is not supported by the V1 Engine. Falling back to V0. \n", + "Unsloth: vLLM Bitsandbytes config using kwargs = {'load_in_8bit': False, 'load_in_4bit': True, 'bnb_4bit_compute_dtype': 'bfloat16', 'bnb_4bit_quant_storage': 'uint8', 'bnb_4bit_quant_type': 'nf4', 'bnb_4bit_use_double_quant': True, 'llm_int8_enable_fp32_cpu_offload': False, 'llm_int8_has_fp16_weight': False, 'llm_int8_skip_modules': ['lm_head', 'multi_modal_projector', 'merger', 'modality_projection', 'model.layers.4.mlp', 'model.layers.0.mlp', 'model.layers.60.mlp', 'model.layers.62.mlp', 'model.layers.5.mlp', 'model.layers.43.self_attn'], 'llm_int8_threshold': 6.0}\n", + "INFO 03-30 08:37:24 [llm_engine.py:241] Initializing a V0 LLM engine (v0.8.2) with config: model='unsloth/qwq-32b-unsloth-bnb-4bit', speculative_config=None, tokenizer='unsloth/qwq-32b-unsloth-bnb-4bit', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, override_neuron_config=None, tokenizer_revision=None, trust_remote_code=False, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=bitsandbytes, tensor_parallel_size=1, pipeline_parallel_size=1, disable_custom_all_reduce=False, quantization=bitsandbytes, enforce_eager=False, kv_cache_dtype=auto, device_config=cuda:0, decoding_config=DecodingConfig(guided_decoding_backend='xgrammar', reasoning_backend=None), observability_config=ObservabilityConfig(show_hidden_metrics=False, otlp_traces_endpoint=None, collect_model_forward_time=False, collect_model_execute_time=False), seed=0, served_model_name=unsloth/qwq-32b-unsloth-bnb-4bit, num_scheduler_steps=1, multi_step_stream_outputs=True, enable_prefix_caching=True, chunked_prefill_enabled=False, use_async_output_proc=True, disable_mm_preprocessor_cache=False, mm_processor_kwargs=None, pooler_config=None, compilation_config={\"level\":0,\"splitting_ops\":[],\"compile_sizes\":[],\"cudagraph_capture_sizes\":[128,120,112,104,96,88,80,72,64,56,48,40,32,24,16,8,4,2,1],\"max_capture_size\":128}, use_cached_outputs=False, \n", + "INFO 03-30 08:37:24 [cuda.py:291] Using Flash Attention backend.\n", + "INFO 03-30 08:37:25 [parallel_state.py:954] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, TP rank 0\n", + "INFO 03-30 08:37:25 [model_runner.py:1110] Starting to load model unsloth/qwq-32b-unsloth-bnb-4bit...\n", + "INFO 03-30 08:37:25 [loader.py:1155] Loading weights with BitsAndBytes quantization. May take a while ...\n", + "INFO 03-30 08:37:25 [weight_utils.py:265] Using model weights format ['*.safetensors']\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "198d11b20e02412cb2833c5042d2af97", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Loading safetensors checkpoint shards: 0% Completed | 0/5 [00:00\\n\\n\\n\\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n\\n \\n \\n \\n \\n\\n \\n \\n \\n \\n \\n \\n \\n \\n\\n'},\n", + " {'role': 'user',\n", + " 'content': '\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\nAt node \"evaluate_test_results\". Entry 45f3d2 shows failed test: \"Error: Connection timeout\". \\nWill need to check system logs soon (noted in /tasks/reminders.txt, check at 14:00).\\nFirst focusing on this error.\\n\\n```\\n\\n## Reasons to Switch Procedures\\n\\nCommon triggers:\\n - Data available on stdin\\n - Time matching scheduled task\\n - Error conditions in script output\\n - Resource constraints detected\\n - User input needed]]>\\n \\n \\n \\n \\n \\n \\n Also load the last 10 messages from /user/conversation_history/ ]\\n PrepareForDraft{Have everything needed for drafting a message?}\\n DraftMessage[Draft message in reasoning entry]\\n ReadInput[Read input from standard input]\\n ReasonCleanContext[List id\\'s of entries that are no longer needed
Explain for each entry why it is no longer needed]\\n DeleteEntries[Remove the entries that are no longer needed
End by deleting the ReasonCleanContext entry]\\n AddHistoryUser[Add the message to the /user/conversation_history/ directory
The filename is the id of the stdin entry with .user extension]\\n LoadTask[Look for the task in the /tasks directory and load relevant files]\\n LoadUserDetails[Look in the /user directory for relevant files]\\n EstimateScript[Draft the script in a reasoning block and estimate its runtime and output length]\\n ScriptAcceptable{Does the draft script make sense and are the estimations short enough to not hinder the conversation?}\\n RunScript[Run the script, make sure to set appropriate timeout and output limits]\\n ReviewDraft{Is the message well structured and free of logical errors?}\\n SendMessage[Send the message using standard output]\\n AddHistoryAgent[Add the message to the /user/conversation_history/ directory
The filename is the id of the stdout entry with .agent extension]\\n ReasonResponse[Is the conversation ongoing?
How long is the user expected to take to respond?]\\n NeedAwaitResponse{Is it likely to get a response within a minute?}\\n BusyWait[Wait 1 second for the first busy wait, double the time each iteration until a response is received
Make sure to set the timout]\\n\\n End([Clean the context])\\n\\n Start --> LoadUserBasic\\n LoadUserBasic --> PrepareForDraft\\n\\n PrepareForDraft -->|Got all needed info| DraftMessage\\n PrepareForDraft -->|Getting the required info would slow the conversation| DraftMessage\\n PrepareForDraft -->|Input available on stdin| ReadInput\\n PrepareForDraft -->|Context usage more than 50%| ReasonCleanContext\\n PrepareForDraft -->|Task mentioned but not loaded| LoadTask\\n PrepareForDraft -->|Personal or social info mentioned but not loaded| LoadUserDetails\\n PrepareForDraft -->|Calculations, system info or other numerical values that can be scripted are mentioned| EstimateScript\\n\\n ReasonCleanContext --> DeleteEntries\\n DeleteEntries --> PrepareForDraft\\n\\n ReadInput --> AddHistoryUser\\n AddHistoryUser --> PrepareForDraft\\n\\n LoadTask --> PrepareForDraft\\n LoadUserDetails --> PrepareForDraft\\n\\n EstimateScript --> ScriptAcceptable\\n ScriptAcceptable -->|Acceptable| RunScript\\n ScriptAcceptable -->|Not acceptable| PrepareForDraft\\n RunScript --> PrepareForDraft\\n \\n DraftMessage --> ReviewDraft{Is this really what I want to say?}\\n ReviewDraft -->|Rewrite better| DraftMessage\\n ReviewDraft -->|Good message| SendMessage\\n \\n SendMessage --> AddHistoryAgent\\n AddHistoryAgent --> ReasonResponse\\n ReasonResponse --> NeedAwaitResponse\\n \\n NeedAwaitResponse -->|Quick response is unlikely| End\\n NeedAwaitResponse -->|Input available on stdin| PrepareForDraft\\n NeedAwaitResponse -->|Quick response is likely| BusyWait\\n BusyWait --> NeedAwaitResponse\\n```]]>\\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n \\n'},\n", + " {'role': 'assistant', 'content': ''}]}" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dataset[5]" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -139,9 +292,17 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "num_proc must be <= 20. Reducing num_proc to 20 for dataset of size 20.\n" + ] + } + ], "source": [ "trainer = SFTTrainer(\n", " model=model,\n", @@ -155,23 +316,106 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 12, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n", + " \\\\ /| Num examples = 20 | Num Epochs = 3 | Total steps = 3\n", + "O^O/ \\_/ \\ Batch size per device = 1 | Gradient accumulation steps = 16\n", + "\\ / Data Parallel GPUs = 1 | Total batch size (1 x 16 x 1) = 16\n", + " \"-____-\" Trainable parameters = 536,870,912/32,000,000,000 (1.68% trained)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Will smartly offload gradients to save VRAM!\n" + ] + }, + { + "data": { + "text/html": [ + "\n", + "
\n", + " \n", + " \n", + " [3/3 01:45, Epoch 1/3]\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining Loss

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=3, training_loss=1.7514243125915527, metrics={'train_runtime': 187.9691, 'train_samples_per_second': 0.319, 'train_steps_per_second': 0.016, 'total_flos': 7927521441988608.0, 'train_loss': 1.7514243125915527})" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "trainer.train()" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Merging 4bit and LoRA weights to 16bit...\n", + "Unsloth: Will use up to 312.94 out of 503.54 RAM for saving.\n", + "Unsloth: Saving model... This might take 5 minutes ...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 12%|█▎ | 8/64 [00:00<00:02, 22.74it/s]\n", + "We will save to Disk and not RAM now.\n", + "100%|██████████| 64/64 [00:43<00:00, 1.48it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Unsloth: Saving tokenizer... Done.\n", + "Done.\n" + ] + } + ], "source": [ "model.save_pretrained_merged(\n", " str(args.output_dir), \n", " tokenizer=tokenizer,\n", - " save_method=\"merged_16bit\"\n", + " #save_method=\"merged_4bit_forced\"\n", ")" ] }