Unloading models + current evals

data/code_eval_board.csv

@@ -29,4 +29,8 @@ T,Models,ARC,HellaSwag,MMLU,TruthfulQA,Winogrande,GSM8K,Reference Model
 🔶,chargoddard/loyal-piano-m7,0.11,0.13,0.19,0.45,0.0,0.97,mistralai/Mistral-7B-v0.1
 🔶,rishiraj/CatPPT,0.09,0.12,0.19,0.44,0.0,0.98,mistralai/Mistral-7B-v0.1
 🔶,togethercomputer/RedPajama-INCITE-Instruct-3B-v1,0.08,0.12,0.19,0.43,0.0,0.77,mistralai/Mistral-7B-v0.1 
-
+🔶,jan-hq/trinity-v1,0.07,0.16,0.18,0.35,0.0,0.95,mistralai/Mistral-7B-v0.1
+🔶,lmsys/vicuna-7b-v1.5,0.13,0.16,0.22,0.62,0.0,0.96,mistralai/Mistral-7B-v0.1
+🟢,huggyllama/llama-7b,0.11,0.17,0.22,0.46,0.0,0.79,mistralai/Mistral-7B-v0.1
+🟢,tiiuae/falcon-7b-instruct,0.06,0.16,0.19,0.56,0.0,0.98,mistralai/Mistral-7B-v0.1
+🔶,NousResearch/Nous-Hermes-llama-2-7b,0.09,0.18,0.26,0.5,0.0,0.96,mistralai/Mistral-7B-v0.1

data/queue.csv

@@ -1,12 +1,7 @@
 Type,Model,ref_model
-🔶 finetuned,lmsys/vicuna-7b-v1.5,mistralai/Mistral-7B-v0.1
-🔶 finetuned,jan-hq/trinity-v1,mistralai/Mistral-7B-v0.1
-🔶 finetuned,microsoft/Orca-2-7b,huggyllama/llama-7b
-🟢 base,huggyllama/llama-7b,mistralai/Mistral-7B-v0.1
 🔶 finetuned,openaccess-ai-collective/DPOpenHermes-7B-v2,mistralai/Mistral-7B-v0.1
-🟢 base,tiiuae/falcon-7b-instruct,mistralai/Mistral-7B-v0.1
+🔶 finetuned,microsoft/Orca-2-7B,mistralai/Mistral-7B-v0.1
 🟢 base,01-ai/Yi-6B,mistralai/Mistral-7B-v0.1
-🔶 finetuned,NousResearch/Nous-Hermes-llama-2-7b,mistralai/Mistral-7B-v0.1
 🔶 finetuned,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,mistralai/Mistral-7B-v0.1
 🔶 finetuned,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,huggyllama/llama-7b
 🔶 finetuned,VAGOsolutions/SauerkrautLM-SOLAR-Instruct,upstage/SOLAR-10.7B-v1.0

detect-pretrain-code-contamination/src/run.py

@@ -37,10 +37,7 @@ def load_data(filename):
         
     return loaded_data
 
-def unload_model(model,tokenizer):
-    print("[X] Cannot unload model! Functionality not implemented!")
-
-def load_model(name1,ref_model):
+def load_model(name1):
     if name1 not in models:
         model1 = AutoModelForCausalLM.from_pretrained(name1, return_dict=True, device_map='auto')
         model1.eval()
@@ -120,7 +117,7 @@ def evaluate_data(test_data, col_name, target_model, ref_model, ratio_gen, data_
         neighbors_dls = load_data(f'saves/{ref_model_clean}/{data_name_clean}/neighbors_dls.txt')
     except:
         ### MODEL 2 likelihoods
-        model2, tokenizer2 = load_model(ref_model,ref_model)
+        model2, tokenizer2 = load_model(ref_model)
         inference2_pass = [] #0: p_ref, #1: all_prob_ref, #2: p_ref_likelihood
         for ex in tqdm(test_data): 
             text = ex[col_name]
@@ -136,14 +133,22 @@ def evaluate_data(test_data, col_name, target_model, ref_model, ratio_gen, data_
             new_ex = get_neighbors(text,inference2_pass[counter][2],model2,tokenizer2,ratio_gen,data_name)
             counter = counter + 1
             neighbors_dls.append(new_ex)
-        unload_model(model2,tokenizer2)
+
+        del models[ref_model]
+        del models[ref_model + "_tokenizer"]
+        model2.cpu()
+        del model2
+        del tokenizer2
+        gc.collect()
+        torch.cuda.empty_cache()
+
         # Because it uses temp it is not invariant, however taking a snapshot in time should be just fine.
         save_data(f'saves/{ref_model_clean}/{data_name_clean}/inference2_pass.txt',inference2_pass)
         save_data(f'saves/{ref_model_clean}/{data_name_clean}/neighbors_dls.txt',neighbors_dls)
         print("Saved ref data, exiting.")
 
     ### MODEL 1 likelihoods
-    model1, tokenizer1 = load_model(target_model,ref_model)
+    model1, tokenizer1 = load_model(target_model)
     inference1_pass = [] #0: p1, #1: all_prob, #2: p1_likelihood, #3: p_lower, #4: p_lower_likelihood
     for ex in tqdm(test_data):
         text = ex[col_name]
@@ -158,7 +163,14 @@ def evaluate_data(test_data, col_name, target_model, ref_model, ratio_gen, data_
         new_ex = RMIA_1(text,inference1_pass[counter][2],inference2_pass[counter][2],model1,tokenizer1,ratio_gen,neighbors_dls[counter])
         counter = counter + 1
         results.append(new_ex)
-    unload_model(model1,tokenizer1)
+    
+    del models[target_model]
+    del models[target_model + "_tokenizer"]
+    model1.cpu()
+    del model1
+    del tokenizer1
+    gc.collect()
+    torch.cuda.empty_cache()
     
     ### Inference ex
     all_output = []

detect-pretrain-code-contamination/src/utils.py

@@ -4,7 +4,7 @@ from torch.nn import CrossEntropyLoss
 
 def evaluate_model(model, tokenizer, dl):
     device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    model = model.to(device)
+    #model = model.to(device)
     losses = []
     for batch in dl:
         batch = tokenizer(batch, padding=True, return_tensors='pt', truncation=True, max_length=150)

requirements.txt

@@ -9,3 +9,5 @@ scikit-learn
 accelerate
 gradio
 plotly
+sentencepiece
+protobuf