falcone 7b

README.md

@@ -0,0 +1,9 @@
+---
+library_name: peft
+---
+## Training procedure
+
+### Framework versions
+
+
+- PEFT 0.4.0

adapter_config.json

@@ -0,0 +1,20 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "tiiuae/falcon-7b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32.0,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "revision": null,
+  "target_modules": [
+    "query_key_value"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f489bc6556f94bee60cae9cc99c9e9b6c36b05dcd75d8dddfb89cda36b5412f0
+size 9460017

all_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.74,
+    "train_loss": 2.2459669013818107,
+    "train_runtime": 103.0683,
+    "train_samples_per_second": 32.425,
+    "train_steps_per_second": 0.233
+}

finetuning_args.json

@@ -0,0 +1,12 @@
+{
+  "finetuning_type": "lora",
+  "lora_alpha": 32.0,
+  "lora_dropout": 0.1,
+  "lora_rank": 8,
+  "lora_target": [
+    "query_key_value"
+  ],
+  "name_module_trainable": "mlp",
+  "num_hidden_layers": 32,
+  "num_layer_trainable": 3
+}

train_results.json

@@ -0,0 +1,7 @@
+{
+    "epoch": 2.74,
+    "train_loss": 2.2459669013818107,
+    "train_runtime": 103.0683,
+    "train_samples_per_second": 32.425,
+    "train_steps_per_second": 0.233
+}

trainer_log.jsonl

@@ -0,0 +1,25 @@
+{"current_steps": 1, "total_steps": 24, "loss": 2.3656, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9786121534345265e-05, "epoch": 0.11, "percentage": 4.17, "elapsed_time": "0:00:04", "remaining_time": "0:01:38"}
+{"current_steps": 2, "total_steps": 24, "loss": 2.3106, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.914814565722671e-05, "epoch": 0.23, "percentage": 8.33, "elapsed_time": "0:00:08", "remaining_time": "0:01:34"}
+{"current_steps": 3, "total_steps": 24, "loss": 2.2921, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.8096988312782174e-05, "epoch": 0.34, "percentage": 12.5, "elapsed_time": "0:00:12", "remaining_time": "0:01:29"}
+{"current_steps": 4, "total_steps": 24, "loss": 2.1831, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.665063509461097e-05, "epoch": 0.46, "percentage": 16.67, "elapsed_time": "0:00:16", "remaining_time": "0:01:24"}
+{"current_steps": 5, "total_steps": 24, "loss": 2.2563, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.4833833507280884e-05, "epoch": 0.57, "percentage": 20.83, "elapsed_time": "0:00:21", "remaining_time": "0:01:20"}
+{"current_steps": 6, "total_steps": 24, "loss": 2.2585, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.267766952966369e-05, "epoch": 0.69, "percentage": 25.0, "elapsed_time": "0:00:25", "remaining_time": "0:01:16"}
+{"current_steps": 7, "total_steps": 24, "loss": 2.3297, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.021903572521802e-05, "epoch": 0.8, "percentage": 29.17, "elapsed_time": "0:00:29", "remaining_time": "0:01:11"}
+{"current_steps": 8, "total_steps": 24, "loss": 2.2337, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.7500000000000003e-05, "epoch": 0.91, "percentage": 33.33, "elapsed_time": "0:00:33", "remaining_time": "0:01:07"}
+{"current_steps": 9, "total_steps": 24, "loss": 2.2176, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.456708580912725e-05, "epoch": 1.03, "percentage": 37.5, "elapsed_time": "0:00:38", "remaining_time": "0:01:03"}
+{"current_steps": 10, "total_steps": 24, "loss": 2.2138, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.147047612756302e-05, "epoch": 1.14, "percentage": 41.67, "elapsed_time": "0:00:42", "remaining_time": "0:00:59"}
+{"current_steps": 11, "total_steps": 24, "loss": 2.23, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.8263154805501297e-05, "epoch": 1.26, "percentage": 45.83, "elapsed_time": "0:00:46", "remaining_time": "0:00:55"}
+{"current_steps": 12, "total_steps": 24, "loss": 2.2902, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5e-05, "epoch": 1.37, "percentage": 50.0, "elapsed_time": "0:00:50", "remaining_time": "0:00:50"}
+{"current_steps": 13, "total_steps": 24, "loss": 2.2732, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.173684519449872e-05, "epoch": 1.49, "percentage": 54.17, "elapsed_time": "0:00:55", "remaining_time": "0:00:47"}
+{"current_steps": 14, "total_steps": 24, "loss": 2.2587, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.852952387243698e-05, "epoch": 1.6, "percentage": 58.33, "elapsed_time": "0:01:00", "remaining_time": "0:00:42"}
+{"current_steps": 15, "total_steps": 24, "loss": 2.1618, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.5432914190872757e-05, "epoch": 1.71, "percentage": 62.5, "elapsed_time": "0:01:04", "remaining_time": "0:00:38"}
+{"current_steps": 16, "total_steps": 24, "loss": 2.2363, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2500000000000006e-05, "epoch": 1.83, "percentage": 66.67, "elapsed_time": "0:01:08", "remaining_time": "0:00:34"}
+{"current_steps": 17, "total_steps": 24, "loss": 2.2207, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.780964274781984e-06, "epoch": 1.94, "percentage": 70.83, "elapsed_time": "0:01:12", "remaining_time": "0:00:29"}
+{"current_steps": 18, "total_steps": 24, "loss": 2.2272, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.3223304703363135e-06, "epoch": 2.06, "percentage": 75.0, "elapsed_time": "0:01:16", "remaining_time": "0:00:25"}
+{"current_steps": 19, "total_steps": 24, "loss": 2.2384, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.166166492719124e-06, "epoch": 2.17, "percentage": 79.17, "elapsed_time": "0:01:21", "remaining_time": "0:00:21"}
+{"current_steps": 20, "total_steps": 24, "loss": 2.1726, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.3493649053890326e-06, "epoch": 2.29, "percentage": 83.33, "elapsed_time": "0:01:24", "remaining_time": "0:00:16"}
+{"current_steps": 21, "total_steps": 24, "loss": 2.1893, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.9030116872178316e-06, "epoch": 2.4, "percentage": 87.5, "elapsed_time": "0:01:29", "remaining_time": "0:00:12"}
+{"current_steps": 22, "total_steps": 24, "loss": 2.2168, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.51854342773295e-07, "epoch": 2.51, "percentage": 91.67, "elapsed_time": "0:01:34", "remaining_time": "0:00:08"}
+{"current_steps": 23, "total_steps": 24, "loss": 2.2572, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.1387846565474045e-07, "epoch": 2.63, "percentage": 95.83, "elapsed_time": "0:01:38", "remaining_time": "0:00:04"}
+{"current_steps": 24, "total_steps": 24, "loss": 2.2699, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 0.0, "epoch": 2.74, "percentage": 100.0, "elapsed_time": "0:01:42", "remaining_time": "0:00:00"}
+{"current_steps": 24, "total_steps": 24, "loss": null, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 2.74, "percentage": 100.0, "elapsed_time": "0:01:42", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,169 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.742857142857143,
+  "global_step": 24,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9786121534345265e-05,
+      "loss": 2.3656,
+      "step": 1
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.914814565722671e-05,
+      "loss": 2.3106,
+      "step": 2
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.8096988312782174e-05,
+      "loss": 2.2921,
+      "step": 3
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 4.665063509461097e-05,
+      "loss": 2.1831,
+      "step": 4
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 4.4833833507280884e-05,
+      "loss": 2.2563,
+      "step": 5
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 4.267766952966369e-05,
+      "loss": 2.2585,
+      "step": 6
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 4.021903572521802e-05,
+      "loss": 2.3297,
+      "step": 7
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 3.7500000000000003e-05,
+      "loss": 2.2337,
+      "step": 8
+    },
+    {
+      "epoch": 1.03,
+      "learning_rate": 3.456708580912725e-05,
+      "loss": 2.2176,
+      "step": 9
+    },
+    {
+      "epoch": 1.14,
+      "learning_rate": 3.147047612756302e-05,
+      "loss": 2.2138,
+      "step": 10
+    },
+    {
+      "epoch": 1.26,
+      "learning_rate": 2.8263154805501297e-05,
+      "loss": 2.23,
+      "step": 11
+    },
+    {
+      "epoch": 1.37,
+      "learning_rate": 2.5e-05,
+      "loss": 2.2902,
+      "step": 12
+    },
+    {
+      "epoch": 1.49,
+      "learning_rate": 2.173684519449872e-05,
+      "loss": 2.2732,
+      "step": 13
+    },
+    {
+      "epoch": 1.6,
+      "learning_rate": 1.852952387243698e-05,
+      "loss": 2.2587,
+      "step": 14
+    },
+    {
+      "epoch": 1.71,
+      "learning_rate": 1.5432914190872757e-05,
+      "loss": 2.1618,
+      "step": 15
+    },
+    {
+      "epoch": 1.83,
+      "learning_rate": 1.2500000000000006e-05,
+      "loss": 2.2363,
+      "step": 16
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 9.780964274781984e-06,
+      "loss": 2.2207,
+      "step": 17
+    },
+    {
+      "epoch": 2.06,
+      "learning_rate": 7.3223304703363135e-06,
+      "loss": 2.2272,
+      "step": 18
+    },
+    {
+      "epoch": 2.17,
+      "learning_rate": 5.166166492719124e-06,
+      "loss": 2.2384,
+      "step": 19
+    },
+    {
+      "epoch": 2.29,
+      "learning_rate": 3.3493649053890326e-06,
+      "loss": 2.1726,
+      "step": 20
+    },
+    {
+      "epoch": 2.4,
+      "learning_rate": 1.9030116872178316e-06,
+      "loss": 2.1893,
+      "step": 21
+    },
+    {
+      "epoch": 2.51,
+      "learning_rate": 8.51854342773295e-07,
+      "loss": 2.2168,
+      "step": 22
+    },
+    {
+      "epoch": 2.63,
+      "learning_rate": 2.1387846565474045e-07,
+      "loss": 2.2572,
+      "step": 23
+    },
+    {
+      "epoch": 2.74,
+      "learning_rate": 0.0,
+      "loss": 2.2699,
+      "step": 24
+    },
+    {
+      "epoch": 2.74,
+      "step": 24,
+      "total_flos": 7.741968380801843e+16,
+      "train_loss": 2.2459669013818107,
+      "train_runtime": 103.0683,
+      "train_samples_per_second": 32.425,
+      "train_steps_per_second": 0.233
+    }
+  ],
+  "max_steps": 24,
+  "num_train_epochs": 3,
+  "total_flos": 7.741968380801843e+16,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f9af0e9e4559a31d903aef0759e2f0c2b2f31bfce96058b0971574956bb33ae
+size 3311