Delete app

app/draw_diagram.py

@@ -1,114 +0,0 @@
-import streamlit as st
-import pandas as pd
-import numpy as np
-from streamlit_echarts import st_echarts
-# from streamlit_echarts import JsCode
-from streamlit_javascript import st_javascript
-# from PIL import Image 
-from app.show_examples import *
-
-links_dic = {}
-
-links_dic = {k.lower().replace('_', '-') : v for k, v in links_dic.items()}
-
-# huggingface_image = Image.open('style/huggingface.jpg')
-
-def nav_to(value):
-    try:
-        url = links_dic[str(value).lower()]
-        js = f'window.open("{url}", "_blank").then(r => window.parent.location.href);'
-        st_javascript(js)
-    except:
-        pass
-
-def draw(folder_name, category_name, dataset_name, metrics):
-    
-    folder = f"./results/{metrics}/"
-
-    display_names = {
-        'SU': 'Speech Understanding',
-        'ASU': 'Audio Scene Understanding',
-        'VU': 'Voice Understanding'
-    }
-    
-    data_path = f'{folder}/{category_name.lower()}.csv'
-    chart_data = pd.read_csv(data_path).round(2)
-
-    # if sorted == 'Ascending':
-    #     ascend = True 
-    # else:
-    #     ascend = False
-
-    new_dataset_name = dataset_name.replace('-', '_').lower()
-    chart_data = chart_data[['Model', new_dataset_name]]
-    
-    chart_data = chart_data.sort_values(by=[new_dataset_name], ascending=True).dropna(axis=0)
-
-    if len(chart_data) == 0:
-        return
-    
-    min_value = round(chart_data.iloc[:, 1::].min().min() - 0.1, 1) 
-    max_value = round(chart_data.iloc[:, 1::].max().max() + 0.1, 1)         
-
-    options = {
-        "title": {"text": f"{display_names[folder_name.upper()]}"},
-        "tooltip": {
-            "trigger": "axis",
-            "axisPointer": {"type": "cross", "label": {"backgroundColor": "#6a7985"}},
-            "triggerOn": 'mousemove',
-        },
-        "legend": {"data": ['Overall Accuracy']},
-        "toolbox": {"feature": {"saveAsImage": {}}},
-        "grid": {"left": "3%", "right": "4%", "bottom": "3%", "containLabel": True},
-        "xAxis": [
-            {
-                "type": "category",
-                "boundaryGap": False,
-                "triggerEvent": True,
-                "data": chart_data['Model'].tolist(),
-            }
-        ],
-        "yAxis": [{"type": "value", 
-                    "min": min_value,
-                    "max": max_value, 
-                    # "splitNumber": 10
-                    }],
-        "series": [{
-                "name": f"{dataset_name}",
-                "type": "line",
-                "data": chart_data[f'{new_dataset_name}'].tolist(),
-            }],
-    }
-    
-    events = {
-        "click": "function(params) { return params.value }"
-    }
-
-    value = st_echarts(options=options, events=events, height="500px")
-    
-    if value != None:
-        # print(value)
-        nav_to(value)
-
-    # if value != None:
-    #     highlight_table_line(value)
-
-    '''
-    Show table
-    '''
-    # st.divider()
-    with st.expander('TABLE'):
-    # chart_data['Link'] = chart_data['Model'].map(links_dic)
-        st.dataframe(chart_data,
-                    #  column_config = {
-                    #      "Link": st.column_config.LinkColumn(
-                    #          display_text= st.image(huggingface_image)
-                    #      ),
-                    #  }, 
-                        hide_index = True, 
-                        use_container_width=True)
-    '''
-    show samples
-    '''
-    show_examples(category_name, dataset_name, chart_data['Model'].tolist())
-    

app/pages.py

@@ -1,303 +0,0 @@
-import streamlit as st
-from app.draw_diagram import *
-
-def dashboard():
-
-    with st.container():
-        st.title("AudioBench")
-   
-        st.markdown("""
-            [gh]: https://github.com/AudioLLMs/AudioBench
-            [![GitHub watchers](https://img.shields.io/github/watchers/AudioLLMs/AudioBench?style=social)][gh]
-            [![GitHub Repo stars](https://img.shields.io/github/stars/AudioLLMs/AudioBench?style=social)][gh]
-            """)
-
-    audio_url = "https://arxiv.org/abs/2406.16020"
-
-    st.divider()
-    st.markdown("#### [AudioBench](%s)" % audio_url)
-    st.markdown("##### :dizzy: A comprehensive evaluation benchmark designed for general instruction-following audiolanguage models")
-    st.markdown('''
-
-                
-                ''')
-
-    with st.container():
-        left_co, center_co, right_co = st.columns([0.5,1, 0.5])
-        with center_co:
-            st.image("./style/audio_overview.png", 
-                     caption="Overview of the datasets in AudioBench.", 
-                     use_column_width = True)
-        
-        st.markdown('''
-
-                
-                ''')
-        
-        st.markdown("###### :dart: Our Benchmark includes: ")
-        cols = st.columns(10)
-        cols[1].metric(label="Tasks", value="8") #delta="Tasks", delta_color="off"
-        cols[2].metric(label="Datasets", value="26")
-        cols[3].metric(label="Test Models", value="5")
-
-        # st.markdown("###### :dart: Supported Models and Datasets: ")
-        
-        # sup = pd.DataFrame(
-        #         {"Dataset": "LibriSpeech-Clean", 
-        #          "Category": st.selectbox('category', ['Speech Understanding']),
-        #          "Task": st.selectbox('task', ['Automatic Speech Recognition']),
-        #          "Metrics": st.selectbox('metrics', ['WER']),
-        #          "Status":True}
-        # )
-        
-        # st.data_editor(sup, num_rows="dynamic")
-
-
-    st.divider()
-    with st.container():
-        st.markdown("##### Citations")
-
-        st.markdown('''
-                    :round_pushpin: AudioBench Paper \n
-                        @article{wang2024audiobench,
-                            title={AudioBench: A Universal Benchmark for Audio Large Language Models},
-                            author={Wang, Bin and Zou, Xunlong and Lin, Geyu and Sun, Shuo and Liu, Zhuohan and Zhang, Wenyu and Liu, Zhengyuan and Aw, AiTi and Chen, Nancy F},
-                            journal={arXiv preprint arXiv:2406.16020},
-                            year={2024}
-                            }
-                    ''')
-
-def asr():
-    st.title("Automatic Speech Recognition")
-    
-    filters_levelone = ['LibriSpeech-Test-Clean', 
-                        'LibriSpeech-Test-Other', 
-                        'Common-Voice-15-En-Test', 
-                        'Peoples-Speech-Test', 
-                        'GigaSpeech-Test', 
-                        'Earnings21-Test', 
-                        'Earnings22-Test', 
-                        'Tedlium3-Test', 
-                        'Tedlium3-Longform-Test', 
-                        'IMDA-Part1-ASR-Test', 
-                        'IMDA-Part2-ASR-Test', 
-                        'IMDA-Part3-ASR-Test', 
-                        'IMDA-Part4-ASR-Test', 
-                        'IMDA-Part5-ASR-Test', 
-                        'IMDA-Part6-ASR-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    
-    # with middle:
-    #     if filter_1 == filters_levelone[0]:
-    #         sort_leveltwo = ['LibriSpeech-Test-Clean', 'LibriSpeech-Test-Other', 'Common-Voice-15-En-Test', 'Peoples-Speech-Test',
-    #                         'GigaSpeech-Test', 'Tedlium3-Test','Tedlium3-Longform-Test', 'Earning-21-Test', 'Earning-22-Test']
-    #     elif filter_1 == filters_levelone[1]:
-    #         sort_leveltwo = ['CN-College-Listen-Test', 'SLUE-P2-SQA5-Test', 'DREAM-TTS-Test', 'Public-SG-SpeechQA-Test']
-        
-    #     elif filter_1 == filters_levelone[2]:
-    #         sort_leveltwo = ['OpenHermes-Audio-Test', 'ALPACA-Audio-Test']
-        
-    #     sort = st.selectbox("Sort Dataset", sort_leveltwo)
-    
-    # with right:
-    #     sorted = st.selectbox('by', ['Ascending', 'Descending'])
-
-    if filter_1:
-        draw('su', 'ASR', filter_1, 'wer')
-    else:
-        draw('su', 'ASR', 'LibriSpeech-Test-Clean', 'wer')
-
-
-    ## examples
-    
-
-def sqa():
-    st.title("Speech Question Answering")
-    
-    binary = ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']
-
-    rest = ['SLUE-P2-SQA5-Test', 
-            'Public-SG-Speech-QA-Test', 
-            'Spoken-Squad-v1']
-
-    filters_levelone = binary + rest
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-
-    if filter_1:
-        if filter_1 in binary:
-            draw('su', 'SQA', filter_1, 'llama3_70b_judge_binary')
-        else:
-            draw('su', 'SQA', filter_1, 'llama3_70b_judge')
-    else:
-        draw('su', 'SQA', 'CN-College-Listen-Test', 'llama3_70b_judge_binary')
-
-def si():
-    st.title("Speech Question Answering")
-    
-    filters_levelone = ['OpenHermes-Audio-Test', 
-                        'ALPACA-Audio-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-
-    if filter_1:
-        draw('su', 'SI', filter_1, 'llama3_70b_judge')
-    else:
-        draw('su', 'SI', 'OpenHermes-Audio-Test', 'llama3_70b_judge')
-
-def ac():
-    st.title("Audio Captioning")
-
-    filters_levelone = ['WavCaps-Test', 
-                        'AudioCaps-Test']
-    filters_leveltwo = ['Llama3-70b-judge', 'Meteor']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    with middle:
-        metric = st.selectbox('Select Metric', filters_leveltwo)
-    
-    # with middle:
-    #     if filter_1 == filters_levelone[0]:
-    #         sort_leveltwo = ['Clotho-AQA-Test', 'WavCaps-QA-Test', 'AudioCaps-QA-Test']
-    #     elif filter_1 == filters_levelone[1]:
-    #         sort_leveltwo = ['WavCaps-Test', 'AudioCaps-Test']
-        
-    #     sort = st.selectbox("Sort Dataset", sort_leveltwo)
-    
-    # with right:
-    #     sorted = st.selectbox('by', ['Ascending', 'Descending'])
-
-    if filter_1 or metric:
-        draw('asu', 'AC',filter_1, metric.lower().replace('-', '_'))
-    else:
-        draw('asu', 'AC', 'WavCaps-Test', 'llama3_70b_judge')
-
-def asqa():
-    st.title("Audio Scene Question Answering")
-
-    filters_levelone = ['Clotho-AQA-Test', 
-                        'WavCaps-QA-Test', 
-                        'AudioCaps-QA-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    
-    if filter_1:
-        draw('asu', 'AQA',filter_1, 'llama3_70b_judge')
-    else:
-        draw('asu', 'AQA', 'Clotho-AQA-Test', 'llama3_70b_judge')
-
-def er():
-    st.title("Emotion Recognition")
-
-    filters_levelone = ['IEMOCAP-Emotion-Test', 
-                        'MELD-Sentiment-Test', 
-                        'MELD-Emotion-Test']
-    sort_leveltwo = []
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    
-    # with middle:
-    #     if filter_1 == filters_levelone[0]:
-    #         sort_leveltwo = ['IEMOCAP-Emotion-Test', 'MELD-Sentiment-Test', 'MELD-Emotion-Test']
-        
-    #     elif filter_1 == filters_levelone[1]:
-    #         sort_leveltwo = ['VoxCeleb1-Accent-Test']
-        
-    #     elif filter_1 == filters_levelone[2]:
-    #         sort_leveltwo = ['VoxCeleb1-Gender-Test', 'IEMOCAP-Gender-Test']
-        
-    #     sort = st.selectbox("Sort Dataset", sort_leveltwo)
-    
-    # with right:
-    #     sorted = st.selectbox('by', ['Ascending', 'Descending'])
-
-    if filter_1:
-        draw('vu', 'ER', filter_1, 'llama3_70b_judge_binary')
-    else:
-        draw('vu', 'ER', 'IEMOCAP-Emotion-Test', 'llama3_70b_judge_binary')
-
-def ar():
-    st.title("Accent Recognition")
-
-    filters_levelone = ['VoxCeleb-Accent-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-
-
-    if filter_1:
-        draw('vu', 'AR', filter_1, 'llama3_70b_judge')
-    else:
-        draw('vu', 'AR', 'VoxCeleb-Accent-Test', 'llama3_70b_judge')
-
-def gr():
-    st.title("Emotion Recognition")
-
-    filters_levelone = ['VoxCeleb-Gender-Test', 
-                        'IEMOCAP-Gender-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    
-    if filter_1:
-        draw('vu', 'GR', filter_1, 'llama3_70b_judge_binary')
-    else:
-        draw('vu', 'GR', 'VoxCeleb1-Gender-Test', 'llama3_70b_judge_binary')
-
-def spt():
-    st.title("Speech Translation")
-
-    filters_levelone = ['Covost2-EN-ID-test', 
-                        'Covost2-EN-ZH-test',
-                        'Covost2-EN-TA-test', 
-                        'Covost2-ID-EN-test', 
-                        'Covost2-ZH-EN-test', 
-                        'Covost2-TA-EN-test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    
-    if filter_1:
-        draw('su', 'ST', filter_1, 'bleu')
-    else:
-        draw('su', 'ST', 'Covost2-EN-ID-test', 'bleu')
-
-def cnasr():
-    st.title("Chinese Automatic Speech Recognition")
-
-    filters_levelone = ['Aishell-ASR-ZH-Test']
-    
-    left, center, _, middle,right = st.columns([0.2, 0.2, 0.2, 0.2 ,0.2])
-    
-    with left:
-        filter_1 = st.selectbox('Select Dataset', filters_levelone)
-    
-    if filter_1:
-        draw('su', 'CNASR', filter_1, 'wer')
-    else:
-        draw('su', 'CNASR', 'Aishell-ASR-ZH-Test', 'wer')

app/show_examples.py

@@ -1,129 +0,0 @@
-import streamlit as st
-import datasets
-import numpy as np
-
-
-def show_examples(category_name, dataset_name, model_lists):
-    st.divider()
-    sample_folder = f"./examples/{category_name}/{dataset_name}"
-    dataset = datasets.load_from_disk(sample_folder)
-
-    for index in range(len(dataset)):
-
-        with st.expander(f'EXAMPLE {index+1}'):
-            col1, col2 = st.columns([0.3, 0.7], vertical_alignment="center")
-
-            with col1:
-                st.audio(f'{sample_folder}/sample_{index}.wav', format="audio/wav")
-                      
-            with col2:
-                with st.container():
-                    custom_css = """
-                                <style>
-                                .my-container-question {
-                                background-color: #F5EEF8;
-                                padding: 10px;
-                                border-radius: 10px;
-                                height: auto;
-                                }
-                                </style>
-                                """
-                    st.markdown(custom_css, unsafe_allow_html=True)
-                    
-                    if dataset_name in ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']:
-                        
-                        choices = dataset[index]['other_attributes']['choices'] 
-                        if isinstance(choices, str):
-                            choices_text = choices
-                        elif isinstance(choices, list):
-                            choices_text = ' '.join(i for i in choices)
-                        
-                        question_text = f"""<div class="my-container-question">
-                                         <p>QUESTION: {dataset[index]['instruction']['text']}</p>
-                                         <p>CHOICES: {choices_text}</p>
-                                         </div>
-                                         """
-                    else:
-                        question_text = f"""<div class="my-container-question">
-                                        <p>QUESTION: {dataset[index]['instruction']['text']}</p>
-                                        </div>"""
-                    
-                    
-                    st.markdown(question_text, unsafe_allow_html=True)
-                
-                with st.container():
-                    custom_css = """
-                                <style>
-                                .my-container-answer {
-                                background-color: #F9EBEA;
-                                padding: 10px;
-                                border-radius: 10px;
-                                height: auto;
-                                }
-                                </style>
-                                """
-                    st.markdown(custom_css, unsafe_allow_html=True)
-                    st.markdown(f"""<div class="my-container-answer">
-                                <p>CORRECT ANSWER: {dataset[index]['answer']['text']}</p>
-                                </div>""", unsafe_allow_html=True)
-
-            
-            st.divider()
-            with st.container():
-                custom_css = """
-                            <style>
-                            .my-container-table {
-                            background-color: #F2F3F4;
-                            padding: 10px;
-                            border-radius: 5px;
-                            # height: 50px;
-                            }
-                            </style>
-                            """
-                st.markdown(custom_css, unsafe_allow_html=True)
-
-                model_lists.sort()
-                
-                s = ''
-                if dataset_name in ['CN-College-Listen-MCQ-Test', 'DREAM-TTS-MCQ-Test']:
-                    for model in model_lists:
-                        try:
-                            s += f"""<tr>
-                                <td>{model}</td>
-                                <td><p>{dataset[index][model]['text']}</p> <p>{choices_text}</p></td>
-                                <td>{dataset[index][model]['model_prediction']}</td>
-                            </tr>"""
-                        except:
-                            print(f"{model} is not in {dataset_name}")
-                            continue
-                else:
-                    for model in model_lists:
-                        try:
-                            s += f"""<tr>
-                                <td>{model}</td>
-                                <td>{dataset[index][model]['text']}</td>
-                                <td>{dataset[index][model]['model_prediction']}</td>
-                            </tr>"""
-                        except:
-                            print(f"{model} is not in {dataset_name}")
-                            continue
-                
-                body_details = f"""<table style="width:100%">
-                <thead>
-                    <tr style="text-align: center;">
-                        <th style="width:20%">MODEL</th>
-                        <th style="width:40%">QUESTION</th>
-                        <th style="width:40%">MODEL PREDICTION</th>
-                    </tr>
-                {s}
-                </thead>
-                </table>"""
-                
-                st.markdown(f"""<div class="my-container-table">
-                                {body_details}
-                                </div>""", unsafe_allow_html=True)
-            
-                st.text("")
-
-    
-