import json import jsonlines from collections import defaultdict def compute_accuracy(input_file: bytes, video_meta_file: str): total_qa_num = 0 total_answered_num = 0 right_num = 0 category_right = defaultdict(float) category_total = defaultdict(float) category_acc = defaultdict(float) model_answers = json.loads(input_file) with jsonlines.open(video_meta_file) as reader: video_meta = list(reader) for meta_data in video_meta: for qa in meta_data['qa']: uid = str(qa["uid"]) if uid in model_answers: total_answered_num += 1 model_answer = model_answers[uid] meta_data['question_type'] = [meta_data['question_type']] if qa["answer"] == "NA": continue for category in meta_data['question_type']: category_total[category] += 1 if model_answer == qa["answer"]: category_right[category] += 1 if model_answer == qa["answer"]: right_num += 1 total_qa_num += 1 for key in category_total: category_acc[key] = category_right[key] / category_total[key] acc = float(right_num) / total_qa_num answered_acc = float(right_num) / total_answered_num category_acc.update({"acc": acc, "answered_acc": answered_acc, "total_qa_num": total_qa_num, "total_answered_num": total_answered_num, "right_num": right_num}) return category_acc