feat: add hidden token button

docs: add message free generation
docs: update formatting FAQ
feat: remove session state message
feat: add interactive pipeline composer

src/distilabel_dataset_generator/apps/faq.py

@@ -1,50 +1,63 @@
 import gradio as gr
 
 with gr.Blocks() as app:
-    gr.Markdown(
-        """### FAQ
+    with gr.Row():
+        with gr.Column(scale=1):
+            pass
+        with gr.Column(scale=3):
+            gr.HTML(
+                """
+                <div style="text-align: justify;">
+                    <div style="text-align: center;">
+                        <img src="https://huggingface.co/spaces/argilla/distilabel-dataset-generator/resolve/main/assets/image.png" alt="Distilabel Dataset Generator" style="width: 300px; margin: 20px auto;">
+                    </div>
 
-        <img src="https://huggingface.co/spaces/argilla/distilabel-dataset-generator/resolve/main/assets/image.png" alt="Distilabel Dataset Generator" style="width: 300px;">
+                    <h4 style="text-align: center;">What is Distilabel Dataset Generator?</h4>
 
-        #### What is Distilabel Dataset Generator?
+                    <p>Distilabel Dataset Generator is a tool that allows you to easily create high-quality datasets for training and fine-tuning language models. It leverages the power of distilabel and advanced language models to generate synthetic data tailored to your specific needs.</p>
 
-        Distilabel Dataset Generator is a tool that allows you to easily create high-quality datasets for training and fine-tuning language models. It leverages the power of distilabel and advanced language models to generate synthetic data tailored to your specific needs.
+                    <p>This tool simplifies the process of creating custom datasets, enabling you to:</p>
+                    <ul>
+                        <li>Define the characteristics of your desired dataset</li>
+                        <li>Generate system prompts automatically</li>
+                        <li>Create sample datasets for quick iteration</li>
+                        <li>Produce full-scale datasets with customizable parameters</li>
+                        <li>Push your generated datasets directly to the Hugging Face Hub</li>
+                    </ul>
 
-        This tool simplifies the process of creating custom datasets, enabling you to:
-        - Define the characteristics of your desired dataset
-        - Generate system prompts automatically
-        - Create sample datasets for quick iteration
-        - Produce full-scale datasets with customizable parameters
-        - Push your generated datasets directly to the Hugging Face Hub
+                    <p>By using Distilabel Dataset Generator, you can rapidly prototype and create datasets for, accelerating your AI development process.</p>
 
-        By using Distilabel Dataset Generator, you can rapidly prototype and create datasets for, accelerating your AI development process.
+                    <h4 style="text-align: center;">How is this free?</h4>
 
-        #### How is this free?
+                    <p>The current implementation is based on <a href="https://huggingface.co/docs/api-inference/index" target="_blank">Free Serverless Hugging Face Inference Endpoints</a>. They are rate limited but free to use for anyone on the Hugging Face Hub. You can re-use the underlying pipeline to generate data with other <a href="https://distilabel.argilla.io/dev/components-gallery/llms/" target="_blank">distilabel LLM integrations</a>.</p>
 
-        The current implementation is based on [Free Serverless Hugging Face Inference Endpoints](https://huggingface.co/docs/api-inference/index). They are rate limited but free to use for anyone on the Hugging Face Hub. You can re-use the underlying pipeline to generate data with other [distilabel LLM integrations](https://distilabel.argilla.io/dev/components-gallery/llms/).
+                    <h4 style="text-align: center;">What is distilabel?</h4>
 
-        #### What is distilabel?
+                    <p>Distilabel is the framework for synthetic data and AI feedback for engineers who need fast, reliable and scalable pipelines based on verified research papers.</p>
 
-        Distilabel is the framework for synthetic data and AI feedback for engineers who need fast, reliable and scalable pipelines based on verified research papers.
+                    <h4 style="text-align: center;">What is synthetic data?</h4>
 
-        #### What is synthetic data?
+                    <p>Synthetic data is data generated by an AI model, instead of being collected from the real world.</p>
 
-        Synthetic data is data generated by an AI model, instead of being collected from the real world.
+                    <h4 style="text-align: center;">What is AI feedback?</h4>
 
-        #### What is AI feedback?
+                    <p>AI feedback is feedback provided by an AI model, instead of being provided by a human.</p>
 
-        AI feedback is feedback provided by an AI model, instead of being provided by a human.
+                    <h4 style="text-align: center;">How is distilabel different from other frameworks?</h4>
 
-        #### How is distilabel different from other synthetic data generation frameworks?
+                    <p>Distilabel can be used for generating synthetic data and AI feedback for a wide variety of projects including traditional predictive NLP (classification, extraction, etc.), or generative and large language model scenarios (instruction following, dialogue generation, judging etc.). Distilabel's programmatic approach allows you to build scalable pipelines for data generation and AI feedback. The goal of distilabel is to accelerate your AI development by quickly generating high-quality, diverse datasets based on verified research methodologies for generating and judging with AI feedback. So, Distilabel is focused and specifically designed to be a tool that for scalable and reliable synthetic data generation.</p>
 
-        Distilabel can be used for generating synthetic data and AI feedback for a wide variety of projects including traditional predictive NLP (classification, extraction, etc.), or generative and large language model scenarios (instruction following, dialogue generation, judging etc.). Distilabel's programmatic approach allows you to build scalable pipelines for data generation and AI feedback. The goal of distilabel is to accelerate your AI development by quickly generating high-quality, diverse datasets based on verified research methodologies for generating and judging with AI feedback. So, Distilabel is focused and specifically designed to be a tool that for scalable and reliable synthetic data generation.
+                    <h4 style="text-align: center;">What do people use distilabel for?</h4>
 
-        #### What do people use distilabel for?
+                    <p>The Argilla community uses distilabel to create amazing <a href="https://huggingface.co/datasets?other=distilabel" target="_blank">datasets</a> and <a href="https://huggingface.co/models?other=distilabel" target="_blank">models</a>.</p>
 
-        The Argilla community uses distilabel to create amazing [datasets](https://huggingface.co/datasets?other=distilabel) and [models](https://huggingface.co/models?other=distilabel).
-
-        - The [1M OpenHermesPreference](https://huggingface.co/datasets/argilla/OpenHermesPreferences) is a dataset of ~1 million AI preferences derived from teknium/OpenHermes-2.5. It shows how we can use distilabel to **synthesize data on an immense scale**.
-        - Our [distilabeled Intel Orca DPO dataset](https://huggingface.co/datasets/argilla/distilabel-intel-orca-dpo-pairs) and the [improved OpenHermes model](https://huggingface.co/argilla/distilabeled-OpenHermes-2.5-Mistral-7B), show how we **improve model performance by filtering out 50%** of the original dataset through **AI feedback**.
-        - The [haiku DPO data](https://github.com/davanstrien/haiku-dpo) outlines how anyone can create a **dataset for a specific task** and **the latest research papers** to improve the quality of the dataset.
-        """
-    )
+                    <ul>
+                        <li>The <a href="https://huggingface.co/datasets/argilla/OpenHermesPreferences" target="_blank">1M OpenHermesPreference</a> is a dataset of ~1 million AI preferences derived from teknium/OpenHermes-2.5. It shows how we can use distilabel to <strong>synthesize data on an immense scale</strong>.</li>
+                        <li>Our <a href="https://huggingface.co/datasets/argilla/distilabel-intel-orca-dpo-pairs" target="_blank">distilabeled Intel Orca DPO dataset</a> and the <a href="https://huggingface.co/argilla/distilabeled-OpenHermes-2.5-Mistral-7B" target="_blank">improved OpenHermes model</a>, show how we <strong>improve model performance by filtering out 50%</strong> of the original dataset through <strong>AI feedback</strong>.</li>
+                        <li>The <a href="https://github.com/davanstrien/haiku-dpo" target="_blank">haiku DPO data</a> outlines how anyone can create a <strong>dataset for a specific task</strong> and <strong>the latest research papers</strong> to improve the quality of the dataset.</li>
+                    </ul>
+                </div>
+                """
+            )
+        with gr.Column(scale=1):
+            pass

src/distilabel_dataset_generator/apps/sft.py

@@ -5,20 +5,20 @@ from typing import Union
 import gradio as gr
 import pandas as pd
 from distilabel.distiset import Distiset
-from huggingface_hub import whoami
 
 from src.distilabel_dataset_generator.pipelines.sft import (
     DEFAULT_DATASET,
     DEFAULT_DATASET_DESCRIPTION,
     DEFAULT_SYSTEM_PROMPT,
     PROMPT_CREATION_PROMPT,
+    generate_pipeline_code,
     get_pipeline,
     get_prompt_generation_step,
 )
 from src.distilabel_dataset_generator.utils import (
     get_login_button,
     get_org_dropdown,
-    get_token
+    get_token,
 )
 
 
@@ -60,13 +60,13 @@ def generate_sample_dataset(system_prompt, progress=gr.Progress()):
 
 
 def generate_dataset(
-    system_prompt,
-    num_turns=1,
-    num_rows=5,
-    private=True,
-    org_name=None,
-    repo_name=None,
-    oauth_token: Union[gr.OAuthToken, None] = None,
+    system_prompt: str,
+    num_turns: int = 1,
+    num_rows: int = 5,
+    private: bool = True,
+    org_name: str = None,
+    repo_name: str = None,
+    oauth_token: str = None,
     progress=gr.Progress(),
 ):
     repo_id = (
@@ -141,18 +141,13 @@ def generate_dataset(
     return pd.DataFrame(outputs)
 
 
-def generate_pipeline_code() -> str:
-    with open("src/distilabel_dataset_generator/pipelines/sft.py", "r") as f:
-        pipeline_code = f.read()
-
-    return pipeline_code
-
 def swap_visibilty(profile: Union[gr.OAuthProfile, None]):
     if profile is None:
-        return gr.update(elem_classes=["main_ui_logged_out"])
+        return gr.update(elem_classes=["main_ui_logged_out"]), gr.Mark
     else:
         return gr.update(elem_classes=["main_ui_logged_in"])
 
+
 css = """
 .main_ui_logged_out{opacity: 0.3; pointer-events: none}
 """
@@ -160,9 +155,16 @@ css = """
 with gr.Blocks(
     title="⚗️ Distilabel Dataset Generator",
     head="⚗️ Distilabel Dataset Generator",
-    css=css
+    css=css,
 ) as app:
-    get_login_button()
+    with gr.Row():
+        with gr.Column(scale=1):
+            get_login_button()
+        with gr.Column(scale=2):
+            gr.Markdown(
+                "This token will only be used to push the dataset to the Hugging Face Hub. It won't be incurring any costs because we are using Free Serverless Inference Endpoints."
+            )
+
     gr.Markdown("## Iterate on a sample dataset")
     with gr.Column() as main_ui:
         dataset_description = gr.TextArea(
@@ -237,6 +239,12 @@ with gr.Blocks(
                 )
 
             with gr.Row(variant="panel"):
+                hf_token = gr.Textbox(
+                    label="Hugging Face Token",
+                    placeholder="hf_...",
+                    type="password",
+                    visible=False,
+                )
                 org_name = get_org_dropdown()
                 repo_name = gr.Textbox(label="Repo name", placeholder="dataset_name")
                 private = gr.Checkbox(
@@ -281,6 +289,7 @@ with gr.Blocks(
             private,
             org_name,
             repo_name,
+            hf_token,
         ],
         outputs=[table],
         show_progress=True,
@@ -294,10 +303,28 @@ with gr.Blocks(
 
     with gr.Accordion("Run this pipeline on Distilabel", open=False):
         pipeline_code = gr.Code(
-            value=generate_pipeline_code(),
+            value=generate_pipeline_code(
+                system_prompt.value, num_turns.value, num_rows.value
+            ),
             language="python",
             label="Distilabel Pipeline Code",
         )
 
+    system_prompt.change(
+        fn=generate_pipeline_code,
+        inputs=[system_prompt, num_turns, num_rows],
+        outputs=[pipeline_code],
+    )
+    num_turns.change(
+        fn=generate_pipeline_code,
+        inputs=[system_prompt, num_turns, num_rows],
+        outputs=[pipeline_code],
+    )
+    num_rows.change(
+        fn=generate_pipeline_code,
+        inputs=[system_prompt, num_turns, num_rows],
+        outputs=[pipeline_code],
+    )
+    app.load(get_token, outputs=[hf_token])
     app.load(get_org_dropdown, outputs=[org_name])
     app.load(fn=swap_visibilty, outputs=main_ui)

src/distilabel_dataset_generator/pipelines/sft.py

@@ -129,13 +129,64 @@ DEFAULT_DATASET = pd.DataFrame(
         ],
     }
 )
+_STOP_SEQUENCES = [
+    "<|eot_id|>",
+    "<|start_header_id|>",
+    "assistant",
+    " \n\n",
+]
 
 
-def get_pipeline(num_turns, num_rows, system_prompt):
+def _get_output_mappings(num_turns):
     if num_turns == 1:
-        output_mappings = {"instruction": "prompt", "response": "completion"}
+        return {"instruction": "prompt", "response": "completion"}
     else:
-        output_mappings = {"conversation": "messages"}
+        return {"conversation": "messages"}
+
+
+def generate_pipeline_code(system_prompt, num_turns, num_rows):
+    input_mappings = _get_output_mappings(num_turns)
+    code = f"""
+from distilabel.pipeline import Pipeline
+from distilabel.steps import KeepColumns
+from distilabel.steps.tasks import MagpieGenerator
+from distilabel.llms import InferenceEndpointsLLM
+
+MODEL = "{MODEL}"
+SYSTEM_PROMPT = "{system_prompt}"
+
+with Pipeline(name="sft") as pipeline:
+    magpie = MagpieGenerator(
+        llm=InferenceEndpointsLLM(
+            model_id=MODEL,
+            tokenizer_id=MODEL,
+            magpie_pre_query_template="llama3",
+            generation_kwargs={{
+                "temperature": 0.8,
+                "do_sample": True,
+                "max_new_tokens": 2048,
+                "stop_sequences": {_STOP_SEQUENCES}
+            }}
+        ),
+        n_turns={num_turns},
+        num_rows={num_rows},
+        system_prompt=SYSTEM_PROMPT,
+        output_mappings={input_mappings},
+    )
+    keep_columns = KeepColumns(
+        columns={list(input_mappings.values())} + ["model_name"],
+    )
+    magpie.connect(keep_columns)
+
+if __name__ == "__main__":
+    distiset = pipeline.run()
+"""
+    return code
+
+
+def get_pipeline(num_turns, num_rows, system_prompt):
+    input_mappings = _get_output_mappings(num_turns)
+    output_mappings = input_mappings
     with Pipeline(name="sft") as pipeline:
         magpie = MagpieGenerator(
             llm=InferenceEndpointsLLM(
@@ -147,13 +198,7 @@ def get_pipeline(num_turns, num_rows, system_prompt):
                     "temperature": 0.8,  # it's the best value for Llama 3.1 70B Instruct
                     "do_sample": True,
                     "max_new_tokens": 2048,
-                    "stop_sequences": [
-                        "<|eot_id|>",
-                        "<|end_of_text|>",
-                        "<|start_header_id|>",
-                        "<|end_header_id|>",
-                        "assistant",
-                    ],
+                    "stop_sequences": _STOP_SEQUENCES,
                 },
             ),
             batch_size=2,
@@ -179,6 +224,7 @@ def get_prompt_generation_step():
                 "temperature": 0.8,
                 "max_new_tokens": 2048,
                 "do_sample": True,
+                "stop_sequences": _STOP_SEQUENCES,
             },
         ),
         use_system_prompt=True,

src/distilabel_dataset_generator/utils.py

@@ -8,7 +8,7 @@ from gradio.oauth import (
 )
 from huggingface_hub import whoami
 
-if (
+_CHECK_IF_SPACE_IS_SET = (
     all(
         [
             OAUTH_CLIENT_ID,
@@ -18,28 +18,19 @@ if (
         ]
     )
     or get_space() is None
-):
+)
+
+if _CHECK_IF_SPACE_IS_SET:
     from gradio.oauth import OAuthToken
 else:
     OAuthToken = str
 
 
 def get_login_button():
-    if (
-        all(
-            [
-                OAUTH_CLIENT_ID,
-                OAUTH_CLIENT_SECRET,
-                OAUTH_SCOPES,
-                OPENID_PROVIDER_URL,
-            ]
-        )
-        or get_space() is None
-    ):
-        return gr.LoginButton(
-            value="Sign in with Hugging Face! (This resets the session state.)",
-            size="lg",
-        )
+    return gr.LoginButton(
+        value="Sign in with Hugging Face!",
+        size="lg",
+    )
 
 
 def get_duplicate_button():
@@ -51,6 +42,7 @@ def list_orgs(oauth_token: OAuthToken = None):
     if oauth_token is None:
         return []
     data = whoami(oauth_token.token)
+    print(data["auth"])
     organisations = [
         entry["entity"]["name"]
         for entry in data["auth"]["accessToken"]["fineGrained"]["scoped"]