llm/SIA
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improved gemma training

Browse Source
This commit is contained in:
Niels Geens
2026-01-11 16:25:40 +01:00
parent 289a52d6ce
commit aee72e65ec
2 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tools/gemma_train/pyproject.toml
View File

@@ -12,6 +12,7 @@ dependencies = [
"bitsandbytes>=0.45.0", "bitsandbytes>=0.45.0",
"datasets==3.3.2", "datasets==3.3.2",
"evaluate==0.4.3", "evaluate==0.4.3",
"kernels>=0.11.1",
"llm_engine_utils @ file:///root/sia/lib/llm_engine_utils", "llm_engine_utils @ file:///root/sia/lib/llm_engine_utils",
"peft==0.13.2", "peft==0.13.2",
"python-dotenv>=1.0.0", "python-dotenv>=1.0.0",

10
tools/gemma_train/src/gemma_train/__main__.py
View File

@@ -37,8 +37,8 @@ def train(config: Config):
dataset = dataset.to_transformers_dataset(tokenizer) dataset = dataset.to_transformers_dataset(tokenizer)
lora_config = LoraConfig( lora_config = LoraConfig(
r=4, r=12,
lora_alpha=4, lora_alpha=24,
target_modules=["q_proj", "o_proj", "k_proj", "v_proj", "gate_proj", "up_proj", "down_proj"], target_modules=["q_proj", "o_proj", "k_proj", "v_proj", "gate_proj", "up_proj", "down_proj"],
lora_dropout=0.05, lora_dropout=0.05,
bias="none", bias="none",
@@ -49,12 +49,12 @@ def train(config: Config):
per_device_train_batch_size=1, per_device_train_batch_size=1,
gradient_accumulation_steps=4, gradient_accumulation_steps=4,
warmup_steps=1, warmup_steps=1,
max_steps=1, num_train_epochs=2,
learning_rate=1e-3, learning_rate=1e-3,
fp16=False, fp16=False,
logging_steps=1, logging_steps=5,
save_strategy="steps", save_strategy="steps",
save_steps=1, save_steps=10,
output_dir=config.output_dir/"lora", output_dir=config.output_dir/"lora",
optim="adamw_torch", optim="adamw_torch",
seed=42, seed=42,