jjkim commited on
Commit
fa82c7f
·
1 Parent(s): e8ced77
Files changed (1) hide show
  1. code_eval.py +4 -4
code_eval.py CHANGED
@@ -180,12 +180,12 @@ class CodeEval(evaluate.Metric):
180
  results = {}
181
  for tid, pred, ref in zip(task_ids, predictions, references):
182
  results[tid] = []
183
- for candidate in pred:
184
- result = Result(task_id=tid, completion_id=len(results))
185
  for test_case in ref:
186
  assert isinstance(test_case, str)
187
  test_program = candidate + "\n" + test_case
188
- args = (test_program, timeout, tid)
189
  future = executor.submit(check_correctness, *args)
190
  result.add(future)
191
  results[tid].append(result)
@@ -278,7 +278,7 @@ class Result(BaseModel):
278
  future.cancel()
279
 
280
  if all(r is not None for r in self.result):
281
- self.passed = all(r["passed"] for r in self.result)
282
 
283
  def done(self):
284
  return self.passed is not None
 
180
  results = {}
181
  for tid, pred, ref in zip(task_ids, predictions, references):
182
  results[tid] = []
183
+ for cid, candidate in enumerate(pred):
184
+ result = Result(task_id=tid, completion_id=cid)
185
  for test_case in ref:
186
  assert isinstance(test_case, str)
187
  test_program = candidate + "\n" + test_case
188
+ args = (test_program, timeout, tid, cid)
189
  future = executor.submit(check_correctness, *args)
190
  result.add(future)
191
  results[tid].append(result)
 
278
  future.cancel()
279
 
280
  if all(r is not None for r in self.result):
281
+ self.passed = all(r == "passed" for r in self.result)
282
 
283
  def done(self):
284
  return self.passed is not None