Add application file

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

app.py

@@ -0,0 +1,211 @@
+import json
+import os
+import shutil
+
+import gradio as gr
+from huggingface_hub import Repository
+
+import openai
+
+HF_TOKEN = os.environ.get("TRL_TOKEN", None)
+API_URL = os.environ.get("API_URL")
+
+theme = gr.themes.Monochrome(
+    primary_hue="indigo",
+    secondary_hue="blue",
+    neutral_hue="slate",
+    radius_size=gr.themes.sizes.radius_sm,
+    font=[
+        gr.themes.GoogleFont("Open Sans"),
+        "ui-sans-serif",
+        "system-ui",
+        "sans-serif",
+    ],
+)
+
+if HF_TOKEN:
+    try:
+        shutil.rmtree("./data/")
+    except:
+        pass
+
+    repo = Repository(
+        local_dir="./data/",
+        clone_from="trl-lib/stack-llama-prompts",
+        use_auth_token=HF_TOKEN,
+        repo_type="dataset",
+    )
+    repo.git_pull()
+
+
+PROMPT_TEMPLATE = """Question: {prompt}\n\nAnswer:"""
+
+def save_inputs_and_outputs(inputs, outputs, generate_kwargs):
+    with open(os.path.join("data", "prompts.jsonl"), "a") as f:
+        json.dump(
+            {"inputs": inputs, "outputs": outputs, "generate_kwargs": generate_kwargs},
+            f,
+            ensure_ascii=False,
+        )
+        f.write("\n")
+        commit_url = repo.push_to_hub()
+
+def generate(
+    instruction,
+    temperature=0.9,
+    max_new_tokens=256,
+    top_p=0.95,
+    repetition_penalty=1.0,
+    do_save=True,
+):
+    output = instruction + str(temperature)
+    s = ""
+    for ch in output:
+        s += ch
+        yield s
+    return s
+    
+example_system_prompt = [
+    "You are a helpful and precise assistant for checking the quality of the answer."
+]
+example_your_prompt = [
+    "[Question]\n{question}\n\n[The Start of Assistant's Answer]\n{answer}\n[The End of of Assistant's Answer]\n\nWe would like to request your feedback on the performance of the AI assistant in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only the value indicating the scores for the Assistant. In the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."
+]
+examples = [
+    ["You are a helpful and precise assistant for checking the quality of the answer.", "[Question]\n{question}\n\n[The Start of Assistant's Answer]\n{answer}\n[The End of of Assistant's Answer]\n\nWe would like to request your feedback on the performance of the AI assistant in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only the value indicating the scores for the Assistant. In the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment."]
+]
+
+
+def process_example(args):
+    for x in generate(args):
+        pass
+    return x
+
+
+def gpt_eval(system_prompt, prompt, question, answer, openai_key):
+    if openai_key is None or len(openai_key) <= 10:
+        yield "Please enter a valid openai API key"
+        return
+    # return prompt.format(question=question, answer=answer)
+    origin_input = prompt.format(question=question, answer=answer)
+    input_str = system_prompt + "\n" + origin_input + "\n\n---\n"
+    yield input_str
+    # return input_str
+    # openai.api_key = "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"
+    openai.api_key = openai_key
+    res = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": origin_input},
+        ],
+        stream=True,
+    )
+    output = ""
+    for chunk in res:
+        content = chunk["choices"][0].get("delta", {}).get("content")
+        if content is not None:
+            # print(content, end="")
+            output += content
+            yield input_str + output
+            
+
+css = ".generating {visibility: hidden}"  # + share_btn_css
+
+
+system_prompt = gr.Textbox(
+    value = "You are a helpful and precise assistant for checking the quality of the answer.",
+    interactive=True,
+    label="System Prompt",
+)
+your_prompt = gr.Textbox(
+    placeholder="Enter your prompt here",
+    value="[Question]\n{question}\n\n[The Start of Assistant's Answer]\n{answer}\n[The End of of Assistant's Answer]\n\nWe would like to request your feedback on the performance of the AI assistant in response to the user question displayed above.\nPlease rate the helpfulness, relevance, accuracy, level of details of their responses. Each assistant receives an overall score on a scale of 1 to 10, where a higher score indicates better overall performance.\nPlease first output a single line containing only the value indicating the scores for the Assistant. In the subsequent line, please provide a comprehensive explanation of your evaluation, avoiding any potential bias and ensuring that the order in which the responses were presented does not affect your judgment.",
+    label="Your Prompt",
+    interactive=True,
+)
+llm_question = gr.Textbox(
+    placeholder="Enter your question here",
+    value="What is the meaning of life?",
+    label="Your Question",
+    elem_id="q-input",
+)
+llm_answer = gr.Textbox(
+    placeholder="Enter your answer here",
+    label="Your LLM's Answer",
+    value="C'est la vie!",
+    elem_id="q-tmp-output",
+)
+
+
+
+with gr.Blocks(theme=theme, analytics_enabled=False, css=css) as demo:
+    with gr.Column():
+        gr.Markdown(
+            """
+            Type in the box below and click the button to generate answers to your most pressing questions!            
+            ⚠️ **Data Collection**: by default, we are collecting the prompts entered in this app to further improve and evaluate the model. Do not share any personal or sensitive information while using the app! You can opt out of this data collection by removing the checkbox below:
+        """
+        )
+        
+        with gr.Row():
+            with gr.Column(scale=3):
+                do_save = gr.Checkbox(
+                    value=True,
+                    label="Store data",
+                    info="You agree to the storage of your prompt and generated text for research and development purposes:",
+                )
+                system_prompt.render()
+                your_prompt.render()
+                llm_question.render()
+                llm_answer.render()
+                
+                with gr.Box():
+                    gr.Markdown("**Evaluation by GPT**")
+                    evaluations = gr.Markdown(elem_id="q-output")
+                submit = gr.Button("Generate", variant="primary")
+                
+
+            with gr.Column(scale=1):
+                openai_key = gr.Textbox(
+                    placeholder="This will not be saved or shared.",
+                    label="OpenAI API",
+                    type="password",
+                )
+                
+                openai_model = gr.Textbox(
+                    value="gpt-3.5-turbo",
+                    label="Model (More opions coming soon) )",
+                )
+    
+                # gr.Examples(
+                #     examples=example_system_prompt,
+                #     inputs=[system_prompt],
+                #     label="Example System Prompt"
+                # )
+                
+                # gr.Examples(
+                #     examples=example_your_prompt,
+                #     inputs=[your_prompt],
+                #     label="Example Your Prompt"
+                # )
+
+        example_box = gr.Examples(
+            examples=examples,
+            inputs=[system_prompt, your_prompt],
+            cache_examples=False,
+        )
+        
+    submit.click(
+        gpt_eval,
+        inputs=[system_prompt, your_prompt, llm_question, llm_answer, openai_key],
+        outputs=[evaluations],
+    )
+  
+    # llm_question.submit(
+    #     generate,
+    #     inputs=[llm_question, temperature, max_new_tokens, top_p, repetition_penalty],
+    #     outputs=[evaluations],
+    # )
+
+demo.queue().launch(debug=True)

requirements.txt

@@ -0,0 +1,7 @@
+gradio
+openai
+torch
+transformers 
+datasets 
+evaluate
+scikit-learn