jjkim
commited on
Commit
·
fa82c7f
1
Parent(s):
e8ced77
- code_eval.py +4 -4
code_eval.py
CHANGED
@@ -180,12 +180,12 @@ class CodeEval(evaluate.Metric):
|
|
180 |
results = {}
|
181 |
for tid, pred, ref in zip(task_ids, predictions, references):
|
182 |
results[tid] = []
|
183 |
-
for candidate in pred:
|
184 |
-
result = Result(task_id=tid, completion_id=
|
185 |
for test_case in ref:
|
186 |
assert isinstance(test_case, str)
|
187 |
test_program = candidate + "\n" + test_case
|
188 |
-
args = (test_program, timeout, tid)
|
189 |
future = executor.submit(check_correctness, *args)
|
190 |
result.add(future)
|
191 |
results[tid].append(result)
|
@@ -278,7 +278,7 @@ class Result(BaseModel):
|
|
278 |
future.cancel()
|
279 |
|
280 |
if all(r is not None for r in self.result):
|
281 |
-
self.passed = all(r
|
282 |
|
283 |
def done(self):
|
284 |
return self.passed is not None
|
|
|
180 |
results = {}
|
181 |
for tid, pred, ref in zip(task_ids, predictions, references):
|
182 |
results[tid] = []
|
183 |
+
for cid, candidate in enumerate(pred):
|
184 |
+
result = Result(task_id=tid, completion_id=cid)
|
185 |
for test_case in ref:
|
186 |
assert isinstance(test_case, str)
|
187 |
test_program = candidate + "\n" + test_case
|
188 |
+
args = (test_program, timeout, tid, cid)
|
189 |
future = executor.submit(check_correctness, *args)
|
190 |
result.add(future)
|
191 |
results[tid].append(result)
|
|
|
278 |
future.cancel()
|
279 |
|
280 |
if all(r is not None for r in self.result):
|
281 |
+
self.passed = all(r == "passed" for r in self.result)
|
282 |
|
283 |
def done(self):
|
284 |
return self.passed is not None
|