Shekswess commited on
Commit
e5e6a27
·
1 Parent(s): 2d17c93

Old generation code

Browse files
Files changed (1) hide show
  1. app.py +2 -63
app.py CHANGED
@@ -7,43 +7,6 @@ import gradio as gr
7
  from synthgenai import DatasetConfig, DatasetGeneratorConfig, LLMConfig, InstructionDatasetGenerator, PreferenceDatasetGenerator,RawDatasetGenerator,SentimentAnalysisDatasetGenerator, SummarizationDatasetGenerator, TextClassificationDatasetGenerator
8
 
9
 
10
- def validate_inputs(*args):
11
- """
12
- Validate that all required inputs are filled.
13
-
14
- Args:
15
- *args: The input values to validate.
16
-
17
- Returns:
18
- bool: True if all required inputs are filled, False otherwise.
19
- """
20
- for arg in args:
21
- if not arg:
22
- return False
23
- return True
24
-
25
- stop_event = asyncio.Event()
26
-
27
- def stop_generation():
28
- """
29
- Stop the dataset generation process.
30
- """
31
- stop_event.set()
32
-
33
- def get_hf_token(hf_token):
34
- """
35
- Retrieve the Hugging Face token from the provided input.
36
-
37
- Args:
38
- hf_token (str): The Hugging Face token.
39
-
40
- Returns:
41
- str: The Hugging Face token.
42
- """
43
- if not hf_token:
44
- raise ValueError("Hugging Face token not provided.")
45
- return hf_token
46
-
47
  def generate_synthetic_dataset(
48
  llm_model,
49
  temperature,
@@ -80,32 +43,11 @@ def generate_synthetic_dataset(
80
  Returns:
81
  str: A message indicating the result of the dataset generation.
82
  """
83
-
84
- hf_token = get_hf_token(hf_token)
85
  os.environ["HF_TOKEN"] = hf_token
86
 
87
  for var in llm_env_vars.split(","):
88
- if "=" in var:
89
- key, value = var.split("=", 1)
90
- os.environ[key.strip()] = value.strip()
91
-
92
- # Validate inputs
93
- if not validate_inputs(
94
- llm_model,
95
- temperature,
96
- top_p,
97
- max_tokens,
98
- dataset_type,
99
- topic,
100
- domains,
101
- language,
102
- num_entries,
103
- hf_token,
104
- hf_repo_name,
105
- llm_env_vars,
106
- ):
107
- return "All fields except API Base and API Key must be filled."
108
-
109
 
110
  llm_config = LLMConfig(
111
  model=llm_model,
@@ -142,11 +84,8 @@ def generate_synthetic_dataset(
142
  else:
143
  return "Invalid dataset type"
144
 
145
-
146
  dataset = asyncio.run(generator.agenerate_dataset())
147
  dataset.save_dataset(hf_repo_name=hf_repo_name)
148
-
149
-
150
  return "Dataset generated and saved successfully."
151
 
152
  def ui_main():
 
7
  from synthgenai import DatasetConfig, DatasetGeneratorConfig, LLMConfig, InstructionDatasetGenerator, PreferenceDatasetGenerator,RawDatasetGenerator,SentimentAnalysisDatasetGenerator, SummarizationDatasetGenerator, TextClassificationDatasetGenerator
8
 
9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
  def generate_synthetic_dataset(
11
  llm_model,
12
  temperature,
 
43
  Returns:
44
  str: A message indicating the result of the dataset generation.
45
  """
 
 
46
  os.environ["HF_TOKEN"] = hf_token
47
 
48
  for var in llm_env_vars.split(","):
49
+ key, value = var.split("=")
50
+ os.environ[key.strip()] = value.strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
51
 
52
  llm_config = LLMConfig(
53
  model=llm_model,
 
84
  else:
85
  return "Invalid dataset type"
86
 
 
87
  dataset = asyncio.run(generator.agenerate_dataset())
88
  dataset.save_dataset(hf_repo_name=hf_repo_name)
 
 
89
  return "Dataset generated and saved successfully."
90
 
91
  def ui_main():