app.py

import gradio as gr
from diffusers import AudioLDMControlNetPipeline, ControlNetModel
import os
from pretty_midi import PrettyMIDI
from tempfile import _TemporaryFileWrapper
import torch
import torchaudio
import logging

logging.basicConfig(level=logging.INFO)

SAMPLE_RATE = 16000

if torch.cuda.is_available():
    device = "cuda"
    torch_dtype = torch.float16
else:
    device = "cpu"
    torch_dtype = torch.float32

controlnet = ControlNetModel.from_pretrained(
    "lauraibnz/midi-audioldm-v2", torch_dtype=torch_dtype
)
pipe = AudioLDMControlNetPipeline.from_pretrained(
    "cvssp/audioldm-m-full", controlnet=controlnet, torch_dtype=torch_dtype
)
pipe = pipe.to(device)
generator = torch.Generator(device)


def predict(
    midi_file=None,
    midi_synth=None,
    prompt="",
    neg_prompt="",
    duration=None,
    seed=0,
    cond=1,
    inf=20,
    guidance_scale=2.5,
    guess=False,
):
    try:
        if isinstance(midi_file, _TemporaryFileWrapper):
            midi_file = midi_file.name
        midi = PrettyMIDI(midi_file)
        if not duration or duration == 0:
            duration = midi_synth[1].shape[0] / SAMPLE_RATE
        if not prompt and not neg_prompt:
            guess_mode = True
        audio = pipe(
            prompt,
            negative_prompt=neg_prompt,
            midi=midi,
            audio_length_in_s=duration,
            num_inference_steps=inf,
            controlnet_conditioning_scale=float(cond),
            guess_mode=guess,
            generator=generator.manual_seed(int(seed)),
            guidance_scale=float(guidance_scale),
        )
        return (SAMPLE_RATE, audio.audios.T)
    except Exception as e:
        logging.error(f"Error in predict function: {str(e)}")
        raise gr.Error(f"An error occurred: {str(e)}")


def synthesize(midi_file=None):
    if isinstance(midi_file, _TemporaryFileWrapper):
        midi_file = midi_file.name
    midi = PrettyMIDI(midi_file)
    midi_synth = midi.synthesize(fs=SAMPLE_RATE)
    midi_synth = midi_synth.reshape(midi_synth.shape[0], 1)
    return (SAMPLE_RATE, midi_synth)


def run_example(
    midi_file=None,
    prompt="",
    neg_prompt="",
    duration=None,
    seed=0,
    cond=1,
    inf=20,
    guidance_scale=2.5,
    guess=False,
):
    midi_synth = synthesize(midi_file)
    gen_audio = predict(
        midi_file,
        midi_synth,
        prompt,
        neg_prompt,
        duration,
        seed,
        cond,
        inf,
        guidance_scale,
        guess,
    )
    return midi_synth, gen_audio


with gr.Blocks(
    title="🎹 MIDI-AudioLDM",
    theme=gr.themes.Base(
        text_size=gr.themes.sizes.text_md, font=[gr.themes.GoogleFont("Nunito Sans")]
    ),
) as demo:
    gr.HTML(
        """
        <h1 align="center"; size="16">🎹 MIDI-AudioLDM </h1>
        """
    )
    gr.Markdown(
        """
            MIDI-AudioLDM is a MIDI-conditioned text-to-audio model based on the project [AudioLDM](https://huggingface.co/spaces/haoheliu/audioldm-text-to-audio-generation). The model has been conditioned using the ControlNet architecture and has been developed within Hugging Face’s [🧨 Diffusers](https://huggingface.co/docs/diffusers/) framework. Once trained, MIDI-AudioLDM accepts a MIDI file and a text prompt as input and returns an audio file, which is an interpretation of the MIDI based on the given text description. This enables detailed control over different musical aspects such as notes, mood and timbre.
            """
    )
    with gr.Column(variant="panel"):
        midi = gr.File(label="midi file", file_types=[".mid"])
        prompt = gr.Textbox(
            label="prompt",
            info="Enter a descriptive text prompt to guide the audio generation.",
        )
        with gr.Row():
            with gr.Column():
                midi_synth = gr.Audio(label="synthesized midi")
                midi.upload(synthesize, midi, midi_synth)
            with gr.Column():
                audio = gr.Audio(label="generated audio")
        with gr.Accordion("Advanced Settings", open=False):
            duration = gr.Slider(
                0,
                20,
                step=2.5,
                label="duration",
                info="Modify the duration in seconds of the output audio file. If not set it will be determined by the MIDI file.",
            )
            inf = gr.Slider(
                0,
                100,
                value=40,
                step=1,
                label="inference steps",
                info="Edit the number of denoising steps. A larger number usually leads to higher quality but slower results.",
            )
            guidance_scale = gr.Slider(
                0,
                4,
                value=2.5,
                step=0.5,
                label="guidance scale",
                info="Modify the guidance scale. The higher the value the more linked the generated audio to the text prompt, sometimes at the expense of lower quality.",
            )
            neg_prompt = gr.Textbox(
                label="negative prompt",
                info="Optionally enter a negative text prompt not to guide the audio generation.",
            )
            seed = gr.Number(
                value=48,
                label="random seed",
                info="Change the random seed for a different generation result.",
            )
            cond = gr.Slider(
                0.0,
                1.0,
                value=1.0,
                step=0.1,
                label="conditioning scale",
                info="Choose a value between 0 and 1. The larger the more it will take the conditioning into account. Lower values are recommended for more creative prompts.",
            )
            guess = gr.Checkbox(
                label="guess mode",
                info="Optionally select guess mode. If so, the model will try to recognize the content of the MIDI without the need of a text prompt.",
            )
    btn = gr.Button("Generate")
    btn.click(
        predict,
        inputs=[
            midi,
            midi_synth,
            prompt,
            neg_prompt,
            duration,
            seed,
            cond,
            inf,
            guidance_scale,
            guess,
        ],
        outputs=[audio],
    )
    gr.Examples(
        examples=[
            ["S00.mid", "piano", "", 10, 48, 1.0, 20, 2.5, False],
            ["S00.mid", "violin", "", 10, 48, 1.0, 20, 2.5, False],
            [
                "S00.mid",
                "woman singing, studio recording",
                "noise",
                10,
                48,
                1.0,
                20,
                2.5,
                False,
            ],
            ["S00.mid", "jazz band, clean", "noise", 10, 48, 1.0, 20, 2.5, False],
            ["S00.mid", "choir", "noise, percussion", 10, 48, 1.0, 20, 2.5, False],
        ],
        inputs=[
            midi,
            prompt,
            neg_prompt,
            duration,
            seed,
            cond,
            inf,
            guidance_scale,
            guess,
        ],
        fn=run_example,
        outputs=[midi_synth, audio],
        cache_examples=True,
    )

demo.launch(show_error=True)