Spaces:
Running
Running
Old generation code
Browse files
app.py
CHANGED
@@ -7,43 +7,6 @@ import gradio as gr
|
|
7 |
from synthgenai import DatasetConfig, DatasetGeneratorConfig, LLMConfig, InstructionDatasetGenerator, PreferenceDatasetGenerator,RawDatasetGenerator,SentimentAnalysisDatasetGenerator, SummarizationDatasetGenerator, TextClassificationDatasetGenerator
|
8 |
|
9 |
|
10 |
-
def validate_inputs(*args):
|
11 |
-
"""
|
12 |
-
Validate that all required inputs are filled.
|
13 |
-
|
14 |
-
Args:
|
15 |
-
*args: The input values to validate.
|
16 |
-
|
17 |
-
Returns:
|
18 |
-
bool: True if all required inputs are filled, False otherwise.
|
19 |
-
"""
|
20 |
-
for arg in args:
|
21 |
-
if not arg:
|
22 |
-
return False
|
23 |
-
return True
|
24 |
-
|
25 |
-
stop_event = asyncio.Event()
|
26 |
-
|
27 |
-
def stop_generation():
|
28 |
-
"""
|
29 |
-
Stop the dataset generation process.
|
30 |
-
"""
|
31 |
-
stop_event.set()
|
32 |
-
|
33 |
-
def get_hf_token(hf_token):
|
34 |
-
"""
|
35 |
-
Retrieve the Hugging Face token from the provided input.
|
36 |
-
|
37 |
-
Args:
|
38 |
-
hf_token (str): The Hugging Face token.
|
39 |
-
|
40 |
-
Returns:
|
41 |
-
str: The Hugging Face token.
|
42 |
-
"""
|
43 |
-
if not hf_token:
|
44 |
-
raise ValueError("Hugging Face token not provided.")
|
45 |
-
return hf_token
|
46 |
-
|
47 |
def generate_synthetic_dataset(
|
48 |
llm_model,
|
49 |
temperature,
|
@@ -80,32 +43,11 @@ def generate_synthetic_dataset(
|
|
80 |
Returns:
|
81 |
str: A message indicating the result of the dataset generation.
|
82 |
"""
|
83 |
-
|
84 |
-
hf_token = get_hf_token(hf_token)
|
85 |
os.environ["HF_TOKEN"] = hf_token
|
86 |
|
87 |
for var in llm_env_vars.split(","):
|
88 |
-
|
89 |
-
|
90 |
-
os.environ[key.strip()] = value.strip()
|
91 |
-
|
92 |
-
# Validate inputs
|
93 |
-
if not validate_inputs(
|
94 |
-
llm_model,
|
95 |
-
temperature,
|
96 |
-
top_p,
|
97 |
-
max_tokens,
|
98 |
-
dataset_type,
|
99 |
-
topic,
|
100 |
-
domains,
|
101 |
-
language,
|
102 |
-
num_entries,
|
103 |
-
hf_token,
|
104 |
-
hf_repo_name,
|
105 |
-
llm_env_vars,
|
106 |
-
):
|
107 |
-
return "All fields except API Base and API Key must be filled."
|
108 |
-
|
109 |
|
110 |
llm_config = LLMConfig(
|
111 |
model=llm_model,
|
@@ -142,11 +84,8 @@ def generate_synthetic_dataset(
|
|
142 |
else:
|
143 |
return "Invalid dataset type"
|
144 |
|
145 |
-
|
146 |
dataset = asyncio.run(generator.agenerate_dataset())
|
147 |
dataset.save_dataset(hf_repo_name=hf_repo_name)
|
148 |
-
|
149 |
-
|
150 |
return "Dataset generated and saved successfully."
|
151 |
|
152 |
def ui_main():
|
|
|
7 |
from synthgenai import DatasetConfig, DatasetGeneratorConfig, LLMConfig, InstructionDatasetGenerator, PreferenceDatasetGenerator,RawDatasetGenerator,SentimentAnalysisDatasetGenerator, SummarizationDatasetGenerator, TextClassificationDatasetGenerator
|
8 |
|
9 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
10 |
def generate_synthetic_dataset(
|
11 |
llm_model,
|
12 |
temperature,
|
|
|
43 |
Returns:
|
44 |
str: A message indicating the result of the dataset generation.
|
45 |
"""
|
|
|
|
|
46 |
os.environ["HF_TOKEN"] = hf_token
|
47 |
|
48 |
for var in llm_env_vars.split(","):
|
49 |
+
key, value = var.split("=")
|
50 |
+
os.environ[key.strip()] = value.strip()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
51 |
|
52 |
llm_config = LLMConfig(
|
53 |
model=llm_model,
|
|
|
84 |
else:
|
85 |
return "Invalid dataset type"
|
86 |
|
|
|
87 |
dataset = asyncio.run(generator.agenerate_dataset())
|
88 |
dataset.save_dataset(hf_repo_name=hf_repo_name)
|
|
|
|
|
89 |
return "Dataset generated and saved successfully."
|
90 |
|
91 |
def ui_main():
|