update

app.py

@@ -2,6 +2,7 @@ import streamlit as st
 from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig #, TextIteratorStreamer
 import torch
 import time
+import requests
 
 # from threading import Thread
 
@@ -89,6 +90,9 @@ def generate_and_stream_text(input_ids, generation_config):
     return generated_text
 
 
+def generate_from_api(user_input, generation_config):
+    return requests.post('https://pauljeffrey--sabiyarn-fastapi-app.modal.run/predict', data={"prompt": user_input, "config": generation_config} )
+
 # Text input
 user_input = st.text_area("Enter text below (please, first read the instructions on how to use in the side bar): ", "<prompt> Tell me a story in pidgin <response>:")
 
@@ -97,7 +101,7 @@ if st.button("Generate"):
     if user_input:
         try:
             st.write("**Generated Text Below:**")
-            input_ids = tokenizer(user_input, return_tensors="pt")["input_ids"].to(device)
+            # input_ids = tokenizer(user_input, return_tensors="pt")["input_ids"].to(device)
             full_output = st.empty()
             
             # # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way.
@@ -110,7 +114,8 @@ if st.button("Generate"):
             # full_output.text(generated_text)
             # def stream_text():
             start_time = time.time()
-            generated_text = generate_and_stream_text(input_ids, generation_config)
+            # generated_text = generate_and_stream_text(input_ids, generation_config
+            generated_text = generate_from_api(user_input, generation_config)
             end_time = time.time()
              # Style definition directly within full_output.text()
             # full_output.text("""<div style="width: 100%; height: 300px; overflow-y: auto; border: 1px solid #ddd; padding: 10px;">""")