Create load_models.py

load_models.py

@@ -0,0 +1,24 @@
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# Model checkpoints
+models_checkpoints = [
+    "HuggingFaceTB/SmolLM-135M-Instruct",
+    "HuggingFaceTB/SmolLM-360M",
+    "HuggingFaceTB/SmolLM-360M-Instruct",
+    "HuggingFaceTB/SmolLM-1.7B",
+    "HuggingFaceTB/SmolLM-1.7B-Instruct",
+    "microsoft/Phi-3-mini-4k-instruct",
+    "microsoft/Phi-3-mini-128k-instruct",
+    "Qwen/Qwen2-1.5B-Instruct",
+    "Qwen/Qwen2-0.5B"
+]
+
+# Dictionary to store models and tokenizers
+models_and_tokenizers = {}
+
+# Loading models and tokenizers
+for checkpoint in models_checkpoints:
+    model = AutoModelForCausalLM.from_pretrained(checkpoint)
+    tokenizer = AutoTokenizer.from_pretrained(checkpoint)
+    models_and_tokenizers[checkpoint] = (model, tokenizer)
+