File size: 2,907 Bytes
9becace |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
import time
import os
import json
from werkzeug.utils import secure_filename
import re
import ast
import sqlite3
import random
from transformers import AutoModelForCausalLM, AutoTokenizer
from llmware.models import ModelCatalog
from llmware.prompts import Prompt
def model_test_run_general():
t0 = time.time()
model_name = "llmware/slim-sql-1b-v0"
print("update: model_name - ", model_name)
custom_hf_model = AutoModelForCausalLM.from_pretrained(model_name,trust_remote_code=True)
hf_tokenizer = AutoTokenizer.from_pretrained(model_name)
# now, we have 'imported' our own custom 'instruct' model into llmware
model = ModelCatalog().load_hf_generative_model(custom_hf_model, hf_tokenizer, instruction_following=False,
prompt_wrapper="human_bot")
model.temperature = 0.3
# run direct inference on model
print("\nupdate: Starting Generative Instruct Custom Fine-tuned Test")
t1 = time.time()
print("update: time loading model - ", t1 - t0)
fp = ""
fn = "sql_test_100_simple_s.jsonl"
opened_file = open(os.path.join(fp, fn), "r")
prompt_list = []
for i, rows in enumerate(opened_file):
# print("update: ", i, rows)
rows = json.loads(rows)
new_entry = {"question": rows["question"],
"answer": rows["answer"],
"context": rows["context"]}
prompt_list.append(new_entry)
random.shuffle(prompt_list)
total_response_output = []
perfect_match = 0
for i, entries in enumerate(prompt_list):
prompt = entries["question"]
context = re.sub("[\n\r]","", entries["context"])
context = re.sub("\s+", " ", context)
context = re.sub("\"", "", context)
answer = ""
if "answer" in entries:
answer = entries["answer"]
output = model.inference(prompt, add_context=context, add_prompt_engineering=True)
print("\nupdate: model question - ", prompt)
llm_response = re.sub("['\"]", "", output["llm_response"])
answer = re.sub("['\"]", "", answer)
print("update: model response - ", i, llm_response)
print("update: model gold answer - ", answer)
if llm_response.strip().lower() == answer.strip().lower():
perfect_match += 1
print("update: 100% MATCH")
print("update: perfect match accuracy - ", perfect_match / (i+1))
core_output = {"number": i,
"llm_response": output["llm_response"],
"gold_answer": answer,
"prompt": prompt,
"usage": output["usage"]}
total_response_output.append(core_output)
t2 = time.time()
print("update: total processing time: ", t2-t1)
return total_response_output
output = model_test_run_general()
|