Gemma notebook working on runpod
This commit is contained in:
@@ -2,68 +2,216 @@
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "714f4501",
|
||||
"id": "8cdfeb38",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"```\n",
|
||||
"(notebook) root@a70d062f4f65:~/desktop# cd /\n",
|
||||
"(notebook) root@a70d062f4f65:/# jupyter notebook --ip 0.0.0.0 --port 8080 --allow-root\n",
|
||||
"```"
|
||||
"# Gemma 3 Fine-tuning with 4-bit Quantization and GGUF Export\n",
|
||||
"\n",
|
||||
"This notebook demonstrates how to:\n",
|
||||
"1. Load Gemma 3 model with 4-bit quantization\n",
|
||||
"2. Fine-tune using LoRA\n",
|
||||
"3. Save the LoRA adapters\n",
|
||||
"4. Load and save the model in a format compatible with GGUF conversion\n",
|
||||
"5. Convert to GGUF for use with llama.cpp"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "cdab2518",
|
||||
"id": "b66a8058",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade accelerate bitsandbytes datasets peft torch transformers trl"
|
||||
"\"\"\"\n",
|
||||
"bash\n",
|
||||
"cd /\n",
|
||||
". /root/venvs/notebook/bin/activate\n",
|
||||
"jupyter notebook --ip 0.0.0.0 --port 8080 --allow-root\n",
|
||||
"\"\"\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5a82e319",
|
||||
"id": "661b9851",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!nvidia-smi"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "44641175",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade accelerate"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5938458e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade bitsandbytes"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "69e1dd1e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade datasets"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "0a44fce8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade peft"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6f71aec3",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade safetensors"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "aa68c3b1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade torch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9658742d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade transformers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "63ba3dd9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install --upgrade trl"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "9807f2b0",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"from transformers import AutoTokenizer, AutoModelForCausalLM\n",
|
||||
"import os"
|
||||
"print(torch.cuda.is_available())\n",
|
||||
"print(torch.cuda.device_count())\n",
|
||||
"print(torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"No GPU available\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b74c5471",
|
||||
"id": "142904af",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_id = \"google/gemma-3-1b-it\""
|
||||
"# Import necessary libraries\n",
|
||||
"from peft import LoraConfig, AutoPeftModelForCausalLM\n",
|
||||
"from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig\n",
|
||||
"from trl import SFTTrainer\n",
|
||||
"import gc\n",
|
||||
"import os\n",
|
||||
"import torch\n",
|
||||
"import transformers\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "8ba9ad8e",
|
||||
"id": "12529c73",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Free up GPU memory\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "284bbe5a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Set model ID\n",
|
||||
"model_id = \"google/gemma-3-1b-it\"\n",
|
||||
"#model_id = \"google/gemma-3-12b-it\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1b921940",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Configure 4-bit quantization\n",
|
||||
"bnb_config = BitsAndBytesConfig(\n",
|
||||
" load_in_4bit=True,\n",
|
||||
" bnb_4bit_use_double_quant=True,\n",
|
||||
" bnb_4bit_quant_type=\"nf4\",\n",
|
||||
" bnb_4bit_compute_dtype=torch.bfloat16\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "53a72255",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load tokenizer\n",
|
||||
"tokenizer = AutoTokenizer.from_pretrained(model_id, token=os.environ['SIA_HF_API_KEY'])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e183a2f8",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Load model with 4-bit quantization\n",
|
||||
"model = AutoModelForCausalLM.from_pretrained(\n",
|
||||
" model_id,\n",
|
||||
" torch_dtype=torch.bfloat16,\n",
|
||||
" device_map={\"\": 0},\n",
|
||||
" quantization_config=bnb_config,\n",
|
||||
" device_map=\"auto\",\n",
|
||||
" token=os.environ['SIA_HF_API_KEY'],\n",
|
||||
" attn_implementation='eager',\n",
|
||||
")"
|
||||
@@ -72,7 +220,6 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "143138b4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -82,7 +229,7 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "703eb030",
|
||||
"id": "bdc4f259",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
@@ -94,10 +241,11 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "291f57e6",
|
||||
"id": "6e46920e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define formatting function for dataset\n",
|
||||
"def format_sia_example(example):\n",
|
||||
" return example['messages'].removeprefix(\"<bos>\")"
|
||||
]
|
||||
@@ -105,15 +253,17 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "f01f4125",
|
||||
"id": "d2897b42",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from peft import LoraConfig\n",
|
||||
"\n",
|
||||
"# Configure LoRA\n",
|
||||
"lora_config = LoraConfig(\n",
|
||||
" r=8,\n",
|
||||
" r=16,\n",
|
||||
" lora_alpha=32,\n",
|
||||
" target_modules=[\"q_proj\", \"o_proj\", \"k_proj\", \"v_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
|
||||
" lora_dropout=0.05,\n",
|
||||
" bias=\"none\",\n",
|
||||
" task_type=\"CAUSAL_LM\",\n",
|
||||
")"
|
||||
]
|
||||
@@ -121,27 +271,40 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e2400819",
|
||||
"id": "acf6104c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import transformers\n",
|
||||
"from trl import SFTTrainer\n",
|
||||
"\n",
|
||||
"# Define training arguments\n",
|
||||
"training_args = transformers.TrainingArguments(\n",
|
||||
" per_device_train_batch_size=1,\n",
|
||||
" gradient_accumulation_steps=4,\n",
|
||||
" warmup_steps=1,\n",
|
||||
" max_steps=1,\n",
|
||||
" learning_rate=1e-3,\n",
|
||||
" fp16=True,\n",
|
||||
" logging_steps=1,\n",
|
||||
" save_strategy=\"steps\",\n",
|
||||
" save_steps=1,\n",
|
||||
" output_dir=\"/root/models/notebook_lora\",\n",
|
||||
" optim=\"paged_adamw_8bit\",\n",
|
||||
" seed=42,\n",
|
||||
" group_by_length=True,\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ac0a611a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Initialize the trainer\n",
|
||||
"trainer = SFTTrainer(\n",
|
||||
" model=model,\n",
|
||||
" train_dataset=dataset,\n",
|
||||
" args=transformers.TrainingArguments(\n",
|
||||
" per_device_train_batch_size=1,\n",
|
||||
" gradient_accumulation_steps=1,\n",
|
||||
" warmup_steps=1,\n",
|
||||
" max_steps=1,\n",
|
||||
" learning_rate=1e-3,\n",
|
||||
" fp16=True,\n",
|
||||
" logging_steps=1,\n",
|
||||
" output_dir=\"/root/models/notebook_train\",\n",
|
||||
" optim=\"paged_adamw_8bit\"\n",
|
||||
" ),\n",
|
||||
" args=training_args,\n",
|
||||
" peft_config=lora_config,\n",
|
||||
" formatting_func=format_sia_example,\n",
|
||||
")"
|
||||
@@ -150,41 +313,81 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ce61cf71",
|
||||
"id": "2fe43ca9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Run the training\n",
|
||||
"trainer.train()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b4e9cc7b",
|
||||
"id": "89e3ec9e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model = trainer.model.merge_and_unload()"
|
||||
"# Save the trained LoRA adapter\n",
|
||||
"trainer.model.save_pretrained(\"/root/models/notebook_lora_adapter\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6c429c9a",
|
||||
"id": "58b64dc5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model.save_pretrained(\"/root/models/notebook/\")"
|
||||
"# Clear memory\n",
|
||||
"del trainer.model\n",
|
||||
"del model\n",
|
||||
"torch.cuda.empty_cache()\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "ef091e77",
|
||||
"id": "85e48508",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tokenizer.save_pretrained(\"/root/models/notebook/\")"
|
||||
"# Load the LoRA model with offloading to manage memory\n",
|
||||
"adapted_model = AutoPeftModelForCausalLM.from_pretrained(\n",
|
||||
" \"/root/models/notebook_lora_adapter\",\n",
|
||||
" torch_dtype=torch.float16, # Use float16 for better compatibility\n",
|
||||
" device_map=\"auto\",\n",
|
||||
" offload_folder=\"offload\",\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c1460b89",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Merge the weights\n",
|
||||
"merged_model = adapted_model.merge_and_unload()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4e91a67e",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Save tokenizer first\n",
|
||||
"tokenizer.save_pretrained(\"/root/models/notebook_merged\")\n",
|
||||
"\n",
|
||||
"# Save the merged model\n",
|
||||
"merged_model.save_pretrained(\n",
|
||||
" \"/root/models/notebook_merged\",\n",
|
||||
" safe_serialization=True\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -220,11 +423,14 @@
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "a62a2175",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!python ./llama.cpp/convert_hf_to_gguf.py --outfile /root/models/notebook.gguf --outtype q8_0 /root/models/notebook"
|
||||
"# Convert to GGUF (if merge was successful)\n",
|
||||
"!python ./llama.cpp/convert_hf_to_gguf.py \\\n",
|
||||
" --outfile /root/models/notebook.gguf \\\n",
|
||||
" --outtype q8_0 \\\n",
|
||||
" /root/models/notebook_merged"
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -270,9 +476,9 @@
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "notebook",
|
||||
"display_name": "Python 3 (ipykernel)",
|
||||
"language": "python",
|
||||
"name": "notebook"
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
|
||||
Reference in New Issue
Block a user