Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -15,10 +15,7 @@ model_id = "gpt2"
|
|
15 |
modelgpt2 = GPT2LMHeadModel.from_pretrained(model_id).to(device)
|
16 |
tokenizergpt2 = GPT2TokenizerFast.from_pretrained(model_id)
|
17 |
|
18 |
-
|
19 |
-
words = text.split() # Split the text into a list of words
|
20 |
-
return len(words)
|
21 |
-
|
22 |
def text_to_sentences(text):
|
23 |
clean_text = text.replace('\n', ' ')
|
24 |
return re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', clean_text)
|
@@ -53,19 +50,20 @@ def calculatePerplexity(text):
|
|
53 |
return ppl.item()
|
54 |
|
55 |
def calculatePerplexities(text):
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
66 |
-
|
67 |
-
|
68 |
-
|
|
|
69 |
|
70 |
def input_api_info():
|
71 |
return [gr.inputs.Textbox(placeholder="Copy and paste here...")]
|
|
|
15 |
modelgpt2 = GPT2LMHeadModel.from_pretrained(model_id).to(device)
|
16 |
tokenizergpt2 = GPT2TokenizerFast.from_pretrained(model_id)
|
17 |
|
18 |
+
|
|
|
|
|
|
|
19 |
def text_to_sentences(text):
|
20 |
clean_text = text.replace('\n', ' ')
|
21 |
return re.split(r'(?<=[^A-Z].[.?]) +(?=[A-Z])', clean_text)
|
|
|
50 |
return ppl.item()
|
51 |
|
52 |
def calculatePerplexities(text):
|
53 |
+
with app.app_context():
|
54 |
+
if text is None or len(text) == 0:
|
55 |
+
return ({'error': 'No query provided'})
|
56 |
+
if len(text) > 9400:
|
57 |
+
return ({'error': 'Cannot analyze more than 9400 characters!'})
|
58 |
+
if len(text.split) > 1500:
|
59 |
+
return ({'error': 'Cannot analyze more than 1500 words'})
|
60 |
+
|
61 |
+
sentences = text_to_sentences(text)
|
62 |
+
perplexities = []
|
63 |
+
for sentence in sentences:
|
64 |
+
perplexity = calculatePerplexity(sentence)
|
65 |
+
perplexities.append({"sentence": sentence, "perplexity": perplexity})
|
66 |
+
return (perplexities)
|
67 |
|
68 |
def input_api_info():
|
69 |
return [gr.inputs.Textbox(placeholder="Copy and paste here...")]
|