shlomihod commited on
Commit
201deaf
·
1 Parent(s): 6ea46f6

add references of classification matching

Browse files
Files changed (1) hide show
  1. app.py +3 -0
app.py CHANGED
@@ -231,6 +231,9 @@ def preprocess_output_line(text):
231
  # https://help.openai.com/en/articles/6272941-classifications-transition-guide#h_e63b71a5c8
232
  # Here we take the label from either the *first* or *last* (for CoT) line of the output
233
  # This is not very robust, but it's a start that doesn't requires asking for a structured output such as JSON
 
 
 
234
  def canonize_label(output, annotation_labels, search_row):
235
  assert search_row in SEARCH_ROW_DICT.keys()
236
 
 
231
  # https://help.openai.com/en/articles/6272941-classifications-transition-guide#h_e63b71a5c8
232
  # Here we take the label from either the *first* or *last* (for CoT) line of the output
233
  # This is not very robust, but it's a start that doesn't requires asking for a structured output such as JSON
234
+ # HELM has more robust processing options, we are not using them, but these are the references:
235
+ # https://github.com/stanford-crfm/helm/blob/04a75826ce75835f6d22a7d41ae1487104797964/src/helm/benchmark/metrics/classification_metrics.py
236
+ # https://github.com/stanford-crfm/helm/blob/04a75826ce75835f6d22a7d41ae1487104797964/src/helm/benchmark/metrics/basic_metrics.py
237
  def canonize_label(output, annotation_labels, search_row):
238
  assert search_row in SEARCH_ROW_DICT.keys()
239