Renamed mistral tool to mistral_api

tools/mistral_local_train/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "mistral_local_train"
+version = "0.1.0"
+requires-python = ">=3.8"
+
+dependencies = [
+    "accelerate==1.2.0",
+    "bitsandbytes>=0.45.0",
+    "datasets==3.3.2",
+    "evaluate==0.4.3",
+    "kernels>=0.11.1",
+    "llm_engine_utils @ file:///root/sia/lib/llm_engine_utils",
+    "mistral-common>=1.8.6",
+    "peft==0.13.2",
+    "python-dotenv>=1.0.0",
+    "sentencepiece>=0.2.0",
+    "torch==2.4.1",
+    "transformers>=5.0.0rc0",
+    "trl>=0.17.0",
+]
+
+[project.scripts]
+mistral_local_train = "mistral_local_train.__main__:main"

tools/mistral_local_train/src/mistral_local_train/__main__.py

@@ -0,0 +1,106 @@
+from llm_engine_utils.dataset import Dataset
+from pathlib import Path
+from peft import LoraConfig, AutoPeftModelForCausalLM
+from transformers import AutoTokenizer, TrainingArguments
+from transformers.models.ministral3 import Ministral3ForCausalLM
+from trl import SFTTrainer
+import os
+import sys
+import torch
+
+from .config import Config
+
+def main():
+    config = Config()
+    train(config)
+    merge(config)
+
+def train(config: Config):
+    tokenizer = AutoTokenizer.from_pretrained(
+        config.model,
+        token=config.api_key,
+        trust_remote_code=True,
+        use_fast=False,
+    )
+    tokenizer.save_pretrained(config.output_dir/"tokenizer")
+
+    # Use Ministral3ForCausalLM for text-only training
+    # Note: torch_dtype is deprecated in transformers 5.x, use dtype instead
+    model = Ministral3ForCausalLM.from_pretrained(
+        config.model,
+        dtype=torch.float32,
+        device_map="cpu",
+        token=config.api_key,
+        attn_implementation='eager',
+        low_cpu_mem_usage=True,
+    )
+
+    dataset = Dataset(config.config_path)
+    dataset.validate()
+    dataset = dataset.to_transformers_dataset(tokenizer)
+
+    # LoRA config targeting Ministral-3 architecture layers
+    # Ministral-3 uses alternating attention (1 full + 3 sliding window) across 34 layers
+    # Target modules include:
+    #   - Attention projections: q_proj, k_proj, v_proj, o_proj
+    #   - MLP projections: gate_proj, up_proj, down_proj
+    #   - Language model head: lm_head
+    lora_config = LoraConfig(
+        r=8,
+        lora_alpha=16,
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj", "lm_head"],
+        lora_dropout=0.05,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+
+    training_args = TrainingArguments(
+        per_device_train_batch_size=1,
+        gradient_accumulation_steps=4,
+        warmup_steps=1,
+        num_train_epochs=2,
+        learning_rate=1e-3,
+        fp16=False,
+        logging_steps=5,
+        save_strategy="steps",
+        save_steps=10,
+        output_dir=config.output_dir/"lora",
+        optim="adamw_torch",
+        seed=42,
+        group_by_length=True,
+        use_cpu=True,
+    )
+
+    trainer = SFTTrainer(
+        model=model,
+        train_dataset=dataset,
+        args=training_args,
+        peft_config=lora_config,
+        formatting_func=format_sia_example,
+    )
+
+    trainer.train()
+
+    trainer.model.save_pretrained(config.output_dir/"lora_adapter")
+
+def merge(config: Config):
+    # Note: torch_dtype is deprecated in transformers 5.x, use dtype instead
+    adapted_model = AutoPeftModelForCausalLM.from_pretrained(
+        config.output_dir/"lora_adapter",
+        dtype=torch.float16,
+        token=config.api_key,
+        load_in_4bit=False,
+        load_in_8bit=False,
+    )
+    merged_model = adapted_model.merge_and_unload()
+
+    merged_model.save_pretrained(
+        config.output_dir/"merged",
+        safe_serialization=True
+    )
+
+def format_sia_example(example):
+    return example['messages'].removeprefix("<bos>")
+
+if __name__ == "__main__":
+    exit(main())

tools/mistral_local_train/src/mistral_local_train/config.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+import argparse
+import os
+
+class Config:
+    def __init__(self):
+        parser = argparse.ArgumentParser(
+            description='Train Ministral-3 model locally using LoRA fine-tuning'
+        )
+        parser.add_argument(
+            '--config',
+            type=Path,
+            default=Path('/root/sia/training/config.yaml'),
+            help='Path to config file (default: /root/sia/training/config.yaml)'
+        )
+        parser.add_argument(
+            '--model',
+            type=str,
+            default='mistralai/Ministral-3-8B-Base-2512',
+            help='Base model for fine-tuning (default: mistralai/Ministral-3-8B-Base-2512)'
+        )
+        parser.add_argument(
+            '--api-key',
+            type=str,
+            default=os.environ.get('SIA_HF_API_KEY'),
+            help='Huggingface API key (optional, env: SIA_HF_API_KEY)'
+        )
+        parser.add_argument(
+            '--output-dir',
+            type=Path,
+            default=Path('/root/models/current'),
+            help='Output directory for fine-tuned model and converted gguf (default: /root/models/current)'
+        )
+        self.args = parser.parse_args()
+
+    @property
+    def config_path(self) -> Path:
+        return self.args.config
+
+    @property
+    def model(self) -> str:
+        return self.args.model
+
+    @property
+    def api_key(self) -> str:
+        return self.args.api_key
+
+    @property
+    def output_dir(self) -> Path:
+        return self.args.output_dir