From 95546208e8e7769b76d94c93d23e6d58f0d46fe8 Mon Sep 17 00:00:00 2001 From: Niels Geens Date: Wed, 30 Apr 2025 13:36:24 +0200 Subject: [PATCH] Gemma notebook --- scripts/install.sh | 2 +- tools/notebook/gemma.ipynb | 280 +++++++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 tools/notebook/gemma.ipynb diff --git a/scripts/install.sh b/scripts/install.sh index 7d0341b..2584bc1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -16,7 +16,7 @@ fi if [ -z "${SIA_INSTALL_NO_NOTEBOOK}"]; then echo "Installing venv for running notebooks..." python3 -m venv /root/venvs/notebook - /root/venvs/notebook/bin/pip install ipykernel ipywidgets + /root/venvs/notebook/bin/pip install jupyter ipykernel ipywidgets /root/venvs/notebook/bin/ipython kernel install --name=notebook fi diff --git a/tools/notebook/gemma.ipynb b/tools/notebook/gemma.ipynb new file mode 100644 index 0000000..37fd074 --- /dev/null +++ b/tools/notebook/gemma.ipynb @@ -0,0 +1,280 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "714f4501", + "metadata": {}, + "source": [ + "```\n", + "(notebook) root@a70d062f4f65:~/desktop# cd /\n", + "(notebook) root@a70d062f4f65:/# jupyter notebook --ip 0.0.0.0 --port 8080 --allow-root\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "cdab2518", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install --upgrade accelerate datasets peft torch transformers trl" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "5a82e319", + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "from transformers import AutoTokenizer, AutoModelForCausalLM\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "b74c5471", + "metadata": {}, + "outputs": [], + "source": [ + "model_id = \"google/gemma-3-1b-it\"" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8ba9ad8e", + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer = AutoTokenizer.from_pretrained(model_id, token=os.environ['SIA_HF_API_KEY'])" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e183a2f8", + "metadata": {}, + "outputs": [], + "source": [ + "model = AutoModelForCausalLM.from_pretrained(\n", + " model_id,\n", + " torch_dtype=torch.bfloat16,\n", + " device_map={\"\": 0},\n", + " token=os.environ['SIA_HF_API_KEY'],\n", + " attn_implementation='eager',\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "143138b4", + "metadata": {}, + "outputs": [], + "source": [ + "exec(open(\"/root/sia/tools/train/train/dataset.py\").read())" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "703eb030", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = Dataset(\"/root/sia/training/config.yaml\")\n", + "dataset.validate()\n", + "dataset = dataset.to_transformers_dataset(tokenizer)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "291f57e6", + "metadata": {}, + "outputs": [], + "source": [ + "def format_sia_example(example):\n", + " return example['messages'].removeprefix(\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "f01f4125", + "metadata": {}, + "outputs": [], + "source": [ + "from peft import LoraConfig\n", + "\n", + "lora_config = LoraConfig(\n", + " r=8,\n", + " target_modules=[\"q_proj\", \"o_proj\", \"k_proj\", \"v_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n", + " task_type=\"CAUSAL_LM\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "e2400819", + "metadata": {}, + "outputs": [], + "source": [ + "import transformers\n", + "from trl import SFTTrainer\n", + "\n", + "trainer = SFTTrainer(\n", + " model=model,\n", + " train_dataset=dataset,\n", + " args=transformers.TrainingArguments(\n", + " per_device_train_batch_size=1,\n", + " gradient_accumulation_steps=1,\n", + " warmup_steps=1,\n", + " max_steps=1,\n", + " learning_rate=1e-3,\n", + " fp16=True,\n", + " logging_steps=1,\n", + " output_dir=\"/root/models/notebook_train\",\n", + " optim=\"paged_adamw_8bit\"\n", + " ),\n", + " peft_config=lora_config,\n", + " formatting_func=format_sia_example,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ce61cf71", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "b4e9cc7b", + "metadata": {}, + "outputs": [], + "source": [ + "model = trainer.model.merge_and_unload()" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "6c429c9a", + "metadata": {}, + "outputs": [], + "source": [ + "model.save_pretrained(\"/root/models/notebook/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "ef091e77", + "metadata": {}, + "outputs": [], + "source": [ + "tokenizer.save_pretrained(\"/root/models/notebook/\")" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "30c823ee", + "metadata": {}, + "outputs": [], + "source": [ + "!git clone https://github.com/ggml-org/llama.cpp.git" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "4a1f4b71", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r llama.cpp/requirements.txt" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "51edd868", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install llama-cpp-python" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a62a2175", + "metadata": {}, + "outputs": [], + "source": [ + "!python ./llama.cpp/convert_hf_to_gguf.py --outfile /root/models/notebook.gguf --outtype q8_0 /root/models/notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6fea3615", + "metadata": {}, + "outputs": [], + "source": [ + "!apt update && apt install -y unzip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2418ba6", + "metadata": {}, + "outputs": [], + "source": [ + "!wget https://github.com/ggml-org/llama.cpp/releases/download/b5226/llama-b5226-bin-ubuntu-x64.zip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "262270a6", + "metadata": {}, + "outputs": [], + "source": [ + "unzip llama-b5226-bin-ubuntu-x64.zip" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff438dcc", + "metadata": {}, + "outputs": [], + "source": [ + "!cd build/bin/ && ./llama-cli -m /root/models/notebook.gguf" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "notebook", + "language": "python", + "name": "notebook" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}