add flexibility pipeline, update something

app.py

@@ -4,7 +4,8 @@ from typing import Optional
 import gradio as gr
 import pandas as pd
 
-from utils.similarity import batch_cos_sim
+from utils import pipeline
+from utils.models import list_models
 
 
 def read_data(filepath: str) -> Optional[pd.DataFrame]:
@@ -17,22 +18,45 @@ def read_data(filepath: str) -> Optional[pd.DataFrame]:
     return df
 
 
-def process(model_name: str,
+def process(task_name: str,
+            model_name: str,
             text: str,
             file=None,
             ):
-    if file:
-        df = read_data(file.name)
-    elif text:
-        string_io = StringIO(text)
-        df = pd.read_csv(string_io)
-    else:
-        raise Exception('No input provided')
-    df = batch_cos_sim(df, model_name)
-    path = 'output.csv'
-    df.to_csv(path, index=False, encoding='utf-8-sig')
-    return str(df), path
+    try:
+        # load file
+        if file:
+            df = read_data(file.name)
+        elif text:
+            string_io = StringIO(text)
+            df = pd.read_csv(string_io)
+            assert len(df) >= 1, 'No input data'
+        else:
+            raise Exception('No input data')
+
+        # process
+        if task_name == 'Originality':
+            df = pipeline.p0_originality(df, model_name)
+        elif task_name == 'Flexibility':
+            df = pipeline.p1_flexibility(df, model_name)
+        else:
+            raise Exception('Task not supported')
+
+        # save
+        path = 'output.csv'
+        df.to_csv(path, index=False, encoding='utf-8-sig')
+        return str(df), path
+    except Exception as e:
+        return {'Error': e}, None
+
 
+instructions = 'Please upload a file or paste the text below. '
+
+task_name_dropdown = gr.components.Dropdown(
+    label='Task Name',
+    value='Originality',
+    choices=['Originality', 'Flexibility']
+)
 
 model_name_input = gr.components.Textbox(
     value='paraphrase-multilingual-MiniLM-L12-v2',
@@ -40,18 +64,14 @@ model_name_input = gr.components.Textbox(
     type='text'
 )
 
-model_name_option = gr.components.Dropdown(
+model_name_dropdown = gr.components.Dropdown(
     label='Model Name',
-    value='paraphrase-multilingual-MiniLM-L12-v2',
-    choices=[
-        'paraphrase-multilingual-MiniLM-L12-v2',
-        'paraphrase-multilingual-mpnet-base-v2',
-        'cyclone/simcse-chinese-roberta-wwm-ext'
-    ]
+    value=list_models[0],
+    choices=list_models
 )
 
 text_input = gr.components.Textbox(
-    value='prompt,response\n',
+    value='id,prompt,response\n',
     lines=10,
     type='text'
 )
@@ -61,13 +81,16 @@ text_output = gr.components.Textbox(
     type='text'
 )
 
+description = open('description.txt', 'r').read()
+
 file_output = gr.components.File(label='Output File',
                                  file_count='single',
                                  file_types=['', '.', '.csv', '.xls', '.xlsx'])
 
 app = gr.Interface(
     fn=process,
-    inputs=[model_name_option, text_input, 'file'],
-    outputs=[text_output, file_output]
+    inputs=[task_name_dropdown, model_name_dropdown, text_input, 'file'],
+    outputs=[text_output, file_output],
+    description=description
 )
 app.launch()

description.txt

@@ -0,0 +1,4 @@
+Enter your participant (id) + prompt + response data, one per line, with a COMMA between each variable
+If an error occurred, try simplifying your data - does it work with fewer rows? If not, the input format may be wrong.
+Please note that Sbert_mpnet and Sbert_minilm are applicable to multiple languages, SimCSE is only applicable to Chinese.
+For more help, or to report possible bugs in our system, contact [email protected]

output.csv

@@ -1,13 +1,5 @@
-prompt,response,originality
-床单,当空调被,0.6427325010299683
-床单,保暖,0.5928247570991516
-床单,绑在树上做成吊床,0.5714011490345001
-床单,当燃料烧,0.7625655382871628
-床单,包裹东西,0.41448450088500977
-床单,裁剪成衣服,0.5791812241077423
-牙刷,用来刷首饰,0.5138461589813232
-牙刷,刷鞋,0.5954866111278534
-牙刷,洗水果,0.6339634656906128
-牙刷,捅人,0.5337955951690674
-牙刷,洗马桶,0.5022678673267365
-牙刷,刷桃子的毛,0.6439318358898163
+id,prompt,flexibility
+1,床单,0.60231946905454
+1,牙刷,0.5810987452665964
+2,床单,0.585410421093305
+2,牙刷,0.5599984327952067

utils/models.py

@@ -6,6 +6,12 @@ from sentence_transformers import SentenceTransformer
 
 DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu'
 
+list_models = [
+    'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2',
+    'sentence-transformers/paraphrase-multilingual-mpnet-base-v2',
+    'cyclone/simcse-chinese-roberta-wwm-ext'
+]
+
 
 class SBert:
     def __init__(self, path):

utils/pipeline.py

@@ -0,0 +1,35 @@
+import pandas as pd
+from sentence_transformers.util import cos_sim
+
+from utils.models import SBert
+
+
+def p0_originality(df: pd.DataFrame, model_name: str) -> pd.DataFrame:
+    assert 'prompt' in df.columns
+    assert 'response' in df.columns
+    model = SBert(model_name)
+
+    def get_cos_sim(model, prompt: str, response: str) -> float:
+        prompt_vec = model(prompt)
+        response_vec = model(response)
+        score = cos_sim(prompt_vec, response_vec).item()
+        return score
+
+    df['originality'] = df.apply(lambda x: 1 - get_cos_sim(model, x['prompt'], x['response']), axis=1)
+    return df
+
+
+def p1_flexibility(df: pd.DataFrame, model_name: str) -> pd.DataFrame:
+    df = p0_originality(df, model_name)
+    assert 'id' in df.columns
+    df_out = df.groupby(by=['id', 'prompt']) \
+        .agg({'id': 'first', 'prompt': 'first', 'originality': 'mean'}) \
+        .rename(columns={'originality': 'flexibility'}) \
+        .reset_index(drop=True)
+    return df_out
+
+
+if __name__ == '__main__':
+    _df_input = pd.read_csv('data/example_3.csv')
+    _df_0 = p0_originality(_df_input, 'paraphrase-multilingual-MiniLM-L12-v2')
+    _df_1 = p1_flexibility(_df_input, 'paraphrase-multilingual-MiniLM-L12-v2')

utils/similarity.py

@@ -1,25 +0,0 @@
-import pandas as pd
-from sentence_transformers.util import cos_sim
-
-from utils.models import SBert
-
-
-def get_cos_sim(model, prompt: str, response: str) -> float:
-    prompt_vec = model(prompt)
-    response_vec = model(response)
-    score = cos_sim(prompt_vec, response_vec).item()
-    return score
-
-
-def batch_cos_sim(df: pd.DataFrame, model_name) -> pd.DataFrame:
-    # df.columns = ['prompt', 'response']
-    assert 'prompt' in df.columns
-    assert 'response' in df.columns
-    model = SBert(model_name)
-    df['originality'] = df.apply(lambda x: 1 - get_cos_sim(model, x['prompt'], x['response']), axis=1)
-    return df
-
-
-if __name__ == '__main__':
-    _df = pd.read_csv('data/example_1.csv')
-    _df_o = batch_cos_sim(_df, 'paraphrase-multilingual-MiniLM-L12-v2')