llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Updated notebooks to support 4-bit quantization

Browse Source
This commit is contained in:
Niels Geens
2025-04-23 16:37:10 +00:00
parent 1760532e7f
commit 41c336165f
2 changed files with 589 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

116
sia/llm_engine/qwq_llm_engine.ipynb
View File

@@ -2,9 +2,20 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
"Unsloth: Failed to patch Gemma3ForConditionalGeneration.\n",
"🦥 Unsloth Zoo will now patch everything to make training faster!\n",
"INFO 04-23 16:36:10 [__init__.py:239] Automatically detected platform cuda.\n"
]
}
],
"source": [ "source": [
"# Unsloth should be imported before transformers to ensure all optimizations are applied.\n", "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n",
"from unsloth import FastLanguageModel" "from unsloth import FastLanguageModel"
@@ -12,7 +23,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -24,18 +35,18 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"temperature = 0.6\n", "temperature = 0.6\n",
"model_path = \"/root/models/notebook\"\n", "model_path = \"/root/models/notebook_merged_4bit\"\n",
"xml_schema_text = Path(\"/root/sia/action_schema.xsd\").read_text()" "xml_schema_text = Path(\"/root/sia/action_schema.xsd\").read_text()"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -47,24 +58,74 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"==((====))== Unsloth 2025.3.19: Fast Qwen2 patching. Transformers: 4.51.3. vLLM: 0.8.2.\n",
" \\\\ /| NVIDIA RTX 6000 Ada Generation. Num GPUs = 1. Max memory: 47.5 GB. Platform: Linux.\n",
"O^O/ \\_/ \\ Torch: 2.6.0+cu124. CUDA: 8.9. CUDA Toolkit: 12.4. Triton: 3.2.0\n",
"\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.29.post2. FA2 = False]\n",
" \"-____-\" Free license: http://github.com/unslothai/unsloth\n",
"Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Sliding Window Attention is enabled but not implemented for `eager`; unexpected results may be encountered.\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"validate_env\n",
"device_map sequential\n",
"validate_env\n",
"device_map OrderedDict([('', 0)])\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "a99acfd42fa645abae77144125a734b7",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"# Load model\n", "# Load model\n",
"model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n", "model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n",
" model_path,\n", " model_path,\n",
" load_in_4bit = True, # False for LoRA 16bit\n",
" fast_inference = True, # Enable vLLM fast inference\n",
" gpu_memory_utilization = 0.8, # Reduce if out of memory\n", " gpu_memory_utilization = 0.8, # Reduce if out of memory\n",
")" ")"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Device set to use cuda:0\n"
]
}
],
"source": [ "source": [
"# Create inference pipeline with memory-efficient settings\n", "# Create inference pipeline with memory-efficient settings\n",
"pipeline = pipeline(\n", "pipeline = pipeline(\n",
@@ -77,7 +138,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -86,7 +147,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -99,7 +160,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -112,7 +173,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 10,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -124,7 +185,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 11,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -139,7 +200,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 12,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -148,7 +209,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 13,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -162,9 +223,18 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 14,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"<write_stdout>\n",
"Hello! I'm just a large language model, so I don't have feelings, but thank you for asking. How can I assist you today?</write_stdout><|im_end|>"
]
}
],
"source": [ "source": [
"for text in streamer:\n", "for text in streamer:\n",
" print(text, end=\"\")" " print(text, end=\"\")"
@@ -172,7 +242,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 15,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -185,7 +255,7 @@
"kernelspec": { "kernelspec": {
"display_name": "sia", "display_name": "sia",
"language": "python", "language": "python",
"name": "sia" "name": "python3"
}, },
"language_info": { "language_info": {
"codemirror_mode": { "codemirror_mode": {

519
tools/train/train/qwq.ipynb
View File

@@ -2,9 +2,20 @@
"cells": [ "cells": [
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 1,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
"Unsloth: Failed to patch Gemma3ForConditionalGeneration.\n",
"🦥 Unsloth Zoo will now patch everything to make training faster!\n",
"INFO 04-23 16:23:47 [__init__.py:239] Automatically detected platform cuda.\n"
]
}
],
"source": [ "source": [
"# Unsloth should be imported before transformers to ensure all optimizations are applied.\n", "# Unsloth should be imported before transformers to ensure all optimizations are applied.\n",
"from unsloth import FastLanguageModel, is_bfloat16_supported" "from unsloth import FastLanguageModel, is_bfloat16_supported"
@@ -12,7 +23,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 2,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -28,7 +39,7 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 3,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
@@ -37,18 +48,67 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 4,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"args = qwq.Args([\"--output-dir\", \"/root/models/notebook\"])" "args = qwq.Args([\"--output-dir\", \"/root/models/notebook\", \"--base-model\", \"unsloth/QwQ-32B-bnb-4bit\"])"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 5,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Validating 20 XML files...\n",
"file: /root/sia/training/clean_start/iteration_20250116_134549_655.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/clean_start/iteration_20250116_134549_655.xml\n",
"file: /root/sia/training/clean_start/iteration_20250116_134555_680.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/clean_start/iteration_20250116_134555_680.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141241_092.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141241_092.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141252_317.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141252_317.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141302_940.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141302_940.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141329_886.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141329_886.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141343_416.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141343_416.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141357_412.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141357_412.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141410_965.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141410_965.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141428_204.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141428_204.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141441_443.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141441_443.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141447_231.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141447_231.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141454_509.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141454_509.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141458_495.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141458_495.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141503_889.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141503_889.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141516_718.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141516_718.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141533_231.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141533_231.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141603_549.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141603_549.xml\n",
"file: /root/sia/training/delete_indicated_entries/iteration_20250116_141633_083.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/delete_indicated_entries/iteration_20250116_141633_083.xml\n",
"file: /root/sia/training/list_entries_to_delete/iteration_20250116_141227_271.xml\n",
"WARNING: System prompt hash mismatch in /root/sia/training/list_entries_to_delete/iteration_20250116_141227_271.xml\n",
"Validation complete. Found 20 valid files.\n"
]
}
],
"source": [ "source": [
"dataset = qwq.Dataset(args.config_path)\n", "dataset = qwq.Dataset(args.config_path)\n",
"dataset.validate()" "dataset.validate()"
@@ -56,9 +116,94 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 6,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b72535c55d214b9da158b90ab0d3e65a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer_config.json: 0%| | 0.00/8.14k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5b09cbc50ad94b81a25cf790b5841d75",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"vocab.json: 0%| | 0.00/2.78M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "d561672fb7e34306aada26df664e08cd",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"merges.txt: 0%| | 0.00/1.67M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "b2fa48006e3047689ce3951347ee8d87",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer.json: 0%| | 0.00/11.4M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e4bb1c113f5b46dc95b90924de24aaaf",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"added_tokens.json: 0%| | 0.00/707 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "200953e14bad47a38757dd643c7bb176",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"special_tokens_map.json: 0%| | 0.00/614 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"with open('/root/sia/qwq_tokenizer_config.json', 'r') as f:\n", "with open('/root/sia/qwq_tokenizer_config.json', 'r') as f:\n",
" tokenizer_config = json.load(f)\n", " tokenizer_config = json.load(f)\n",
@@ -71,9 +216,211 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 7,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"==((====))== Unsloth 2025.3.19: Fast Qwen2 patching. Transformers: 4.51.3. vLLM: 0.8.2.\n",
" \\\\ /| NVIDIA RTX 6000 Ada Generation. Num GPUs = 1. Max memory: 47.5 GB. Platform: Linux.\n",
"O^O/ \\_/ \\ Torch: 2.6.0+cu124. CUDA: 8.9. CUDA Toolkit: 12.4. Triton: 3.2.0\n",
"\\ / Bfloat16 = TRUE. FA [Xformers = 0.0.29.post2. FA2 = False]\n",
" \"-____-\" Free license: http://github.com/unslothai/unsloth\n",
"Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "78af66cb4ea84a4e8dea6cb5db0a7ca2",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model.safetensors.index.json: 0%| | 0.00/280k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "8d4904d5f68941369fef4e0b67f21397",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00001-of-00004.safetensors: 0%| | 0.00/4.93G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "728274cc4ab84d64ab152b22a842e1bc",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00002-of-00004.safetensors: 0%| | 0.00/4.96G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "4273150dc40846008cc36d19b87e595e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00003-of-00004.safetensors: 0%| | 0.00/5.00G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0d4ad6c3535741a2aeb454c98d04a395",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"model-00004-of-00004.safetensors: 0%| | 0.00/4.32G [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Sliding Window Attention is enabled but not implemented for `eager`; unexpected results may be encountered.\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "73f1c658797c4b73937e26c2012e2019",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Loading checkpoint shards: 0%| | 0/4 [00:00<?, ?it/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "e926624cc8a1453aae8e0a814c3430da",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"generation_config.json: 0%| | 0.00/238 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "ee12750395cc40b2a44b99e24a52ef23",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer_config.json: 0%| | 0.00/8.14k [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "36c00f147e684cca97857e4f9539b6ed",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"vocab.json: 0%| | 0.00/2.78M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "5bf2c158111a4a5c8f434d1493c6dc98",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"merges.txt: 0%| | 0.00/1.67M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "cb036c9083a24ba092cc2c5d57f0bd32",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"added_tokens.json: 0%| | 0.00/707 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "6fcc331064ac4c2fa67461098f82fff6",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"special_tokens_map.json: 0%| | 0.00/614 [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "7dea89c178f34e2185d7ebbac5b9f84e",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"tokenizer.json: 0%| | 0.00/11.4M [00:00<?, ?B/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n", "model, _returned_tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name = args.base_model,\n", " model_name = args.base_model,\n",
@@ -83,9 +430,17 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 8,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unsloth 2025.3.19 patched 64 layers with 64 QKV layers, 64 O layers and 64 MLP layers.\n"
]
}
],
"source": [ "source": [
"model = FastLanguageModel.get_peft_model(\n", "model = FastLanguageModel.get_peft_model(\n",
" model,\n", " model,\n",
@@ -102,12 +457,12 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 9,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [],
"source": [ "source": [
"training_args = TrainingArguments(\n", "training_args = TrainingArguments(\n",
" output_dir=str(args.output_dir),\n", " output_dir=str(args.output_dir) + \"_train\",\n",
" num_train_epochs=3,\n", " num_train_epochs=3,\n",
" per_device_train_batch_size=1,\n", " per_device_train_batch_size=1,\n",
" gradient_accumulation_steps=16,\n", " gradient_accumulation_steps=16,\n",
@@ -128,9 +483,45 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 10,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "0bf6b9b1d6de48688ff82a8077790f7a",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Generating train split: 0 examples [00:00, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"num_proc must be <= 20. Reducing num_proc to 20 for dataset of size 20.\n"
]
},
{
"data": {
"application/vnd.jupyter.widget-view+json": {
"model_id": "74e69db167224b35b381b91827ac0c30",
"version_major": 2,
"version_minor": 0
},
"text/plain": [
"Unsloth: Tokenizing [\"messages\"] (num_proc=20): 0%| | 0/20 [00:00<?, ? examples/s]"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [ "source": [
"trainer = SFTTrainer(\n", "trainer = SFTTrainer(\n",
" model=model,\n", " model=model,\n",
@@ -143,21 +534,103 @@
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 11,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"==((====))== Unsloth - 2x faster free finetuning | Num GPUs used = 1\n",
" \\\\ /| Num examples = 20 | Num Epochs = 3 | Total steps = 3\n",
"O^O/ \\_/ \\ Batch size per device = 1 | Gradient accumulation steps = 16\n",
"\\ / Data Parallel GPUs = 1 | Total batch size (1 x 16 x 1) = 16\n",
" \"-____-\" Trainable parameters = 134,217,728/32,000,000,000 (0.42% trained)\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Unsloth: Will smartly offload gradients to save VRAM!\n"
]
},
{
"data": {
"text/html": [
"\n",
" <div>\n",
" \n",
" <progress value='3' max='3' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
" [3/3 01:48, Epoch 1/3]\n",
" </div>\n",
" <table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: left;\">\n",
" <th>Step</th>\n",
" <th>Training Loss</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" </tbody>\n",
"</table><p>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/plain": [
"TrainOutput(global_step=3, training_loss=1.7701025009155273, metrics={'train_runtime': 187.628, 'train_samples_per_second': 0.32, 'train_steps_per_second': 0.016, 'total_flos': 1.41437049721344e+16, 'train_loss': 1.7701025009155273})"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [ "source": [
"trainer.train()" "trainer.train()"
] ]
}, },
{ {
"cell_type": "code", "cell_type": "code",
"execution_count": null, "execution_count": 13,
"metadata": {}, "metadata": {},
"outputs": [], "outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Unsloth: Merging 4bit and LoRA weights to 4bit...\n",
"This might take 5 minutes...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"/root/venvs/train/lib/python3.10/site-packages/peft/tuners/lora/bnb.py:351: UserWarning: Merge lora module to 4-bit linear may get different generations due to rounding errors.\n",
" warnings.warn(\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Done.\n",
"Unsloth: Saving tokenizer... Done.\n",
"Unsloth: Saving model... This might take 10 minutes for Llama-7b... Done.\n"
]
}
],
"source": [ "source": [
"model.save_pretrained_merged(\n", "model.save_pretrained_merged(\n",
" str(args.output_dir), \n", " str(args.output_dir) + \"_merged_4bit\", \n",
" tokenizer=tokenizer,\n", " tokenizer=tokenizer,\n",
" save_method=\"merged_4bit_forced\"\n", " save_method=\"merged_4bit_forced\"\n",
")" ")"