import openai, os import gradio as gr import time import boto3 import json import numpy as np import wave import io import os from langchain import OpenAI from langchain.chains import ConversationChain from langchain.memory import ConversationSummaryBufferMemory from langchain.memory import ConversationBufferWindowMemory from langchain.chat_models import ChatOpenAI from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler from langchain.schema import HumanMessage import subprocess from contextlib import closing import asyncio import requests # This example uses aiofile for asynchronous file reads. # It's not a dependency of the project but can be installed # with `pip install aiofile`. import aiofile from amazon_transcribe.client import TranscribeStreamingClient from amazon_transcribe.handlers import TranscriptResultStreamHandler from amazon_transcribe.model import TranscriptEvent from amazon_transcribe.utils import apply_realtime_delay def run_shell_cmd(command): # Run a shell command result = subprocess.run(command, shell=True, capture_output=True, text=True) # Check the command output if result.returncode == 0: print("Command executed successfully") print("Command output:") print(result.stdout) else: print("Command failed") print("Error message:") print(result.stderr) def wav_to_pcm(input_file, output_file): cmd = "ffmpeg -i " + input_file + " -f s16le -ar 16000 -ac 1 -acodec pcm_s16le " + output_file run_shell_cmd(cmd) openai.api_key = os.environ["OPENAI_API_KEY"] did_api_key = os.environ["DID_API_KEY"] avatar_url = "https://create-images-results.d-id.com/DefaultPresenters/Magen_f/image.jpeg" polly = boto3.client('polly', region_name='us-east-1') s3 = boto3.client('s3') transcribe = boto3.client('transcribe') #memory = ConversationSummaryBufferMemory(llm=ChatOpenAI(), max_token_limit=2048) memory = ConversationBufferWindowMemory(k=5) conversation = ConversationChain( llm=OpenAI(streaming=True, callbacks=[StreamingStdOutCallbackHandler()], max_tokens=2048, temperature=0.5), memory=memory, ) SAMPLE_RATE = 16000 BYTES_PER_SAMPLE = 2 CHANNEL_NUMS = 1 AUDIO_PATH = '' CHUNK_SIZE = 1024 * 8 REGION = "us-west-2" transcript_text = '' transcriptions = [] class MyEventHandler(TranscriptResultStreamHandler): def __init__(self, transcript_result_stream): super().__init__(transcript_result_stream) self.transcriptions = [] async def handle_transcript_event(self, transcript_event: TranscriptEvent): # This handler can be implemented to handle transcriptions as needed. # Here's an example to get started. results = transcript_event.transcript.results for result in results: for alt in result.alternatives: print(alt.transcript) transcriptions.append(alt.transcript) async def basic_transcribe(): # Setup up our client with our chosen AWS region client = TranscribeStreamingClient(region=REGION) # Start transcription to generate our async stream stream = await client.start_stream_transcription( language_code="zh-CN", media_sample_rate_hz=SAMPLE_RATE, media_encoding="pcm", ) async def write_chunks(): # NOTE: For pre-recorded files longer than 5 minutes, the sent audio # chunks should be rate limited to match the realtime bitrate of the # audio stream to avoid signing issues. async with aiofile.AIOFile(AUDIO_PATH, "rb") as afp: reader = aiofile.Reader(afp, chunk_size=CHUNK_SIZE) await apply_realtime_delay( stream, reader, BYTES_PER_SAMPLE, SAMPLE_RATE, CHANNEL_NUMS ) await stream.input_stream.end_stream() # Instantiate our handler and start processing events handler = MyEventHandler(stream.output_stream) await asyncio.gather(write_chunks(), handler.handle_events()) # Retrieve the transcriptions from the handler #transcriptions = handler.transcriptions def download_file(bucket_name, object_key, file_path): try: # Download the file from S3 s3.download_file(bucket_name, object_key, file_path) print(f"File downloaded successfully: {file_path}") except Exception as e: print(f"Error downloading file: {str(e)}") def polly_text_to_audio(audio_file_name, text, audio_format): if os.path.exists(audio_file_name): os.remove(audio_file_name) print("output mp3 file deleted successfully.") else: print("output mp3 file does not exist.") polly_response = polly.synthesize_speech( Text=text, OutputFormat=audio_format, SampleRate='16000', VoiceId='Zhiyu', LanguageCode='cmn-CN', Engine='neural', LexiconNames=['tigoCN'] ) # Access the audio stream from the response if "AudioStream" in polly_response: # Note: Closing the stream is important because the service throttles on the # number of parallel connections. Here we are using contextlib.closing to # ensure the close method of the stream object will be called automatically # at the end of the with statement's scope. with closing(polly_response["AudioStream"]) as stream: try: # Open a file for writing the output as a binary stream with open(audio_file_name, "wb") as file: file.write(stream.read()) except IOError as error: # Could not write to file, exit gracefully print(error) sys.exit(-1) else: # The response didn't contain audio data, exit gracefully print("Could not stream audio") sys.exit(-1) def play_s3_voice(text): output_file = "/tmp/response.mp3" polly_text_to_audio(output_file, text, "mp3") # Upload the file to an S3 bucket audio_output_bucket_name = "lingo-audio-materials" audio_output_s3_key = "answers/response.mp3" s3.upload_file(output_file, audio_output_bucket_name, audio_output_s3_key) # Construct the S3 bucket URI s3_uri = f"s3://{audio_output_bucket_name}/{audio_output_s3_key}" print("audio output bucket name:"+audio_output_bucket_name) print("audio output key name:"+audio_output_s3_key) mp3_pre_signed_url = s3.generate_presigned_url('get_object',Params={'Bucket': audio_output_bucket_name,'Key': audio_output_s3_key},ExpiresIn=3600) print("mp3_pre_signed_url:"+mp3_pre_signed_url) current_dir = os.getcwd() print("current dir:"+current_dir) print("output_file_location: "+output_file) return output_file, mp3_pre_signed_url def generate_talk_with_audio(input, avatar_url, api_key = did_api_key): url = "https://api.d-id.com/talks" payload = { "script": { "type": "audio", "audio_url": input }, "config": { "auto_match": "true", "result_format": "mp4" }, "source_url": avatar_url } headers = { "accept": "application/json", "content-type": "application/json", "authorization": "Basic " + api_key } response = requests.post(url, json=payload, headers=headers) return response.json() def get_a_talk(id, api_key = os.environ.get('DID_API_KEY')): url = "https://api.d-id.com/talks/" + id headers = { "accept": "application/json", "authorization": "Basic "+api_key } response = requests.get(url, headers=headers) return response.json() def get_mp4_video(input, avatar_url=avatar_url): response = generate_talk_with_audio(input=input, avatar_url=avatar_url) talk = get_a_talk(response['id']) video_url = "" index = 0 while index < 30: index += 1 if 'result_url' in talk: video_url = talk['result_url'] return video_url else: time.sleep(1) talk = get_a_talk(response['id']) return video_url def predict(input, history=[]): if input is not None: history.append(input) response = conversation.predict(input=input) audio_file, pre_signed_url = play_s3_voice(response) video_url = get_mp4_video(input=pre_signed_url, avatar_url=avatar_url) video_html = f"""""" history.append(response) responses = [(u,b) for u,b in zip(history[::2], history[1::2])] return responses, audio_file, video_html, history else: video_html = f'John Carmack' responses = [(u,b) for u,b in zip(history[::2], history[1::2])] return responses, audio_file, video_html, history def transcribe_func_new(audio): audio_file = open(audio, "rb") wav_file = audio_file.name print("wav_file: "+wav_file) #transcript = openai.Audio.transcribe("whisper-1", audio_file) #return transcript['text'] pcm_file = os.path.splitext(wav_file)[0] + ".pcm" wav_to_pcm(wav_file, pcm_file) if os.path.exists(pcm_file): print("pcm file exists") else: print("pcm file does not exist") AUDIO_PATH=pcm_file loop = asyncio.get_event_loop() loop.run_until_complete(basic_transcribe()) loop.close() transcript_text = transcriptions[-1] print("final transcribe script: "+transcript_text) return transcript_text def transcribe_func_old(audio): audio_file = open(audio, "rb") file_name = audio_file.name #file_directory = os.path.dirname(audio_file.name) print("audio_file: "+file_name) #transcript = openai.Audio.transcribe("whisper-1", audio_file) #return transcript['text'] # Set up the job parameters job_name = "lingo-demo" text_output_bucket = 'lingo-text-material' #this bucket is in us-west-1 text_output_key = 'transcriptions/question.json' text_output_key = 'transcriptions/'+job_name+'.json' language_code = 'zh-CN' # Upload the file to an S3 bucket audio_input_bucket_name = "lingo-audio-material" audio_input_s3_key = "questions/tmp-question-from-huggingface.wav" s3.upload_file(file_name, audio_input_bucket_name, audio_input_s3_key) # Construct the S3 bucket URI s3_uri = f"s3://{audio_input_bucket_name}/{audio_input_s3_key}" response = transcribe.list_transcription_jobs() # Iterate through the jobs and print their names for job in response['TranscriptionJobSummaries']: print(job['TranscriptionJobName']) if job['TranscriptionJobName'] == job_name: response = transcribe.delete_transcription_job(TranscriptionJobName=job_name) print("delete transcribe job response:"+str(response)) # Create the transcription job response = transcribe.start_transcription_job( TranscriptionJobName=job_name, Media={'MediaFileUri': s3_uri}, MediaFormat='wav', LanguageCode=language_code, OutputBucketName=text_output_bucket, OutputKey=text_output_key ) print("start transcribe job response:"+str(response)) job_name = response["TranscriptionJob"]["TranscriptionJobName"] # Wait for the transcription job to complete while True: status = transcribe.get_transcription_job(TranscriptionJobName=job_name)['TranscriptionJob']['TranscriptionJobStatus'] if status in ['COMPLETED', 'FAILED']: break print("Transcription job still in progress...") time.sleep(1) # Get the transcript #transcript = transcribe.get_transcription_job(TranscriptionJobName=job_name) transcript_uri = transcribe.get_transcription_job(TranscriptionJobName=job_name)['TranscriptionJob']['Transcript']['TranscriptFileUri'] print("transcript uri: " + str(transcript_uri)) transcript_file_content = s3.get_object(Bucket=text_output_bucket, Key=text_output_key)['Body'].read().decode('utf-8') print(transcript_file_content) json_data = json.loads(transcript_file_content) # Extract the transcript value transcript_text = json_data['results']['transcripts'][0]['transcript'] return transcript_text def process_audio(audio, history=[]): text = transcribe_func_old(audio) return predict(text, history) with gr.Blocks(css="#chatbot{height:350px} .overflow-y-auto{height:500px}") as demo: chatbot = gr.Chatbot(elem_id="chatbot") state = gr.State([]) with gr.Row(): txt = gr.Textbox(show_label=False, placeholder="Enter text and press enter").style(container=False) with gr.Row(): audio_input = gr.Audio(source="microphone", type="filepath", label="Audio Input") with gr.Row(): audio_output = gr.Audio(type="filepath", label="Audio Output", elem_id="speaker", interactive=False) with gr.Row(): video = gr.HTML(f'John Carmack', live=False) txt.submit(predict, [txt, state], [chatbot, audio_output, video, state]) audio_input.change(process_audio, [audio_input, state], [chatbot, audio_output, video, state]) demo.launch(debug=True)