diff --git a/recipes/quickstart/finetuning/README.md b/recipes/quickstart/finetuning/README.md index bee4db7f565c45e5a9066e748c5089c686093c7c..46d58aa6cfd58ae8387cefb9a3ba29d963556bce 100644 --- a/recipes/quickstart/finetuning/README.md +++ b/recipes/quickstart/finetuning/README.md @@ -54,6 +54,7 @@ It lets us specify the training settings for everything from `model_name` to `da output_dir: str = "PATH/to/save/PEFT/model" freeze_layers: bool = False num_freeze_layers: int = 1 + freeze_LLM_only: bool = False # Freeze self-attention layers in the language_model. Vision model, multi_modal_projector, cross-attention will be fine-tuned quantization: str = None one_gpu: bool = False save_model: bool = True diff --git a/recipes/quickstart/finetuning/finetune_vision_model.md b/recipes/quickstart/finetuning/finetune_vision_model.md index 6f7d64f64c1b2f2183c0478f7c212db8bf48cdf1..d0868796895c71a0b5524b1e4f66bbcc5bb368c3 100644 --- a/recipes/quickstart/finetuning/finetune_vision_model.md +++ b/recipes/quickstart/finetuning/finetune_vision_model.md @@ -18,6 +18,12 @@ For **LoRA finetuning with FSDP**, we can run the following code: ```bash torchrun --nnodes 1 --nproc_per_node 4 recipes/quickstart/finetuning/finetuning.py --enable_fsdp --lr 1e-5 --num_epochs 3 --batch_size_training 2 --model_name meta-llama/Llama-3.2-11B-Vision-Instruct --dist_checkpoint_root_folder ./finetuned_model --dist_checkpoint_folder fine-tuned --use_fast_kernels --dataset "custom_dataset" --custom_dataset.test_split "test" --custom_dataset.file "recipes/quickstart/finetuning/datasets/ocrvqa_dataset.py" --run_validation True --batching_strategy padding --use_peft --peft_method lora ``` + +For **finetuning with LLM freeze using FSDP**, we can run the following code: + +```bash + torchrun --nnodes 1 --nproc_per_node 4 recipes/quickstart/finetuning/finetuning.py --enable_fsdp --lr 1e-5 --num_epochs 3 --batch_size_training 2 --model_name meta-llama/Llama-3.2-11B-Vision-Instruct --dist_checkpoint_root_folder ./finetuned_model --dist_checkpoint_folder fine-tuned --use_fast_kernels --dataset "custom_dataset" --custom_dataset.test_split "test" --custom_dataset.file "recipes/quickstart/finetuning/datasets/ocrvqa_dataset.py" --run_validation True --batching_strategy padding --freeze_LLM_only True +``` **Note**: `--batching_strategy padding` is needed as the vision model will not work with `packing` method. For more details about the finetuning configurations, please read the [finetuning readme](./README.md).