From 4fb3504b33913b5f8810a61658c06fa6bc3f3928 Mon Sep 17 00:00:00 2001
From: "a.hediehloo" <alirezahediehloo@gmail.com>
Date: Thu, 20 Nov 2025 06:48:29 +0000
Subject: [PATCH] qwen train: lora

---
 train/qwen/a.sh           | 11 ++++++++---
 train/qwen/merge_model.py | 24 ++++++++++++++++++++++++
 2 files changed, 32 insertions(+), 3 deletions(-)
 create mode 100644 train/qwen/merge_model.py

diff --git a/train/qwen/a.sh b/train/qwen/a.sh
index b874d66..49617b6 100644
--- a/train/qwen/a.sh
+++ b/train/qwen/a.sh
@@ -2,15 +2,20 @@
 
 
 nproc_per_node=1
+
+INFONCE_USE_BATCH=False \
 CUDA_VISIBLE_DEVICES=0 \
 NPROC_PER_NODE=$nproc_per_node \
 swift sft \
     --model $(pwd)/../../data/models/Qwen3-Embedding-0.6B/model \
     --task_type embedding \
     --model_type qwen3_emb \
-    --train_type full \
+    --train_type lora \
+    --lora_rank 8 \
+    --lora_alpha 16 \
+    --target_modules all-linear \
     --dataset my_local_dataset \
-    --custom_register_path /home/hediehloo/codes/embedding/embedding_model/data/dataset/my_dataset_register.py \
+    --custom_register_path $(pwd)/../../data/dataset/my_dataset_register.py \
     --split_dataset_ratio 0.005 \
     --eval_strategy steps \
     --output_dir output \
@@ -25,4 +30,4 @@ swift sft \
     --loss_type infonce \
     --label_names labels \
     --dataloader_drop_last true \
-    --deepspeed zero3
\ No newline at end of file
+    --deepspeed zero3
diff --git a/train/qwen/merge_model.py b/train/qwen/merge_model.py
new file mode 100644
index 0000000..1ff9091
--- /dev/null
+++ b/train/qwen/merge_model.py
@@ -0,0 +1,24 @@
+import json
+import numpy as np
+import os
+from peft import PeftModel
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+
+def merge(base_model_path, peft_model_path, save_path):
+    base_model = AutoModelForCausalLM.from_pretrained(base_model_path, torch_dtype="bfloat16")
+    ft_model = PeftModel.from_pretrained(base_model, peft_model_path)
+    ft_model = ft_model.merge_and_unload()
+    ft_model.save_pretrained(save_path)
+
+def main():
+    file_path = os.path.dirname(__file__)
+
+    base_model_path = file_path + "/../../data/models/Qwen3-Embedding-0.6B/model"
+    peft_model_path = file_path + "/output/v0-20251118-115015/checkpoint-3434"
+    save_path = file_path + "/output/v0-20251118-115015/merged_checkpoint-3434"
+    merge(base_model_path, peft_model_path, save_path)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file