app.py

import gradio as gr
import PIL.Image
import transformers
from transformers import PaliGemmaForConditionalGeneration, PaliGemmaProcessor
import torch
import os
import string
import functools
import re
import flax.linen as nn
import jax
import jax.numpy as jnp
import numpy as np
import spaces
from PIL import Image

model_id = "mattraj/curacel-transcription-1"
COLORS = ['#4285f4', '#db4437', '#f4b400', '#0f9d58', '#e48ef1']
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = PaliGemmaForConditionalGeneration.from_pretrained(model_id, torch_dtype=torch.bfloat16).eval().to(device)
processor = PaliGemmaProcessor.from_pretrained(model_id)

def resize_and_pad(image, target_dim):
    # Calculate the aspect ratio
    scale_factor = 1
    aspect_ratio = image.width / image.height
    if aspect_ratio > 1:
        # Width is greater than height
        new_width = int(target_dim * scale_factor)
        new_height = int((target_dim / aspect_ratio) * scale_factor)
    else:
        # Height is greater than width
        new_height = int(target_dim * scale_factor)
        new_width = int(target_dim * aspect_ratio * scale_factor)

    resized_image = image.resize((new_width, new_height), Image.LANCZOS)

    # Create a new image with the target dimensions and a white background
    new_image = Image.new("RGB", (target_dim, target_dim), (255, 255, 255))
    new_image.paste(resized_image, ((target_dim - new_width) // 2, (target_dim - new_height) // 2))

    return new_image


###### Transformers Inference
@spaces.GPU
def infer(
        image: PIL.Image.Image,
        text: str,
        max_new_tokens: int
) -> str:
    inputs = processor(text=text, images=resize_and_pad(image, 448), return_tensors="pt", padding="longest", do_convert_rgb=True).to(device).to(dtype=model.dtype)
    generated_ids = model.generate(
        **inputs,
        max_length=2048
    )
    result = processor.decode(generated_ids[0], skip_special_tokens=True)
    return result

######## Demo

INTRO_TEXT = """## Curacel Handwritten Arabic demo\n\n
Finetuned from: google/paligemma-3b-pt-448


Translation model demo at: https://prod.arabic-gpt.ai/

Prompts:
Translate the Arabic to English: {model output}

The following is a diagnosis in Arabic from a medical billing form we need to translate to English. The transcriber is not necessariily accurate so one or more characters or words may be wrong. Given what is written, what is the most likely diagnosis. Think step by step, and think about similar words or mispellings in Arabic. Give multiple arabic diagnoses along with the translation in English for each, then finally select the diagnosis that makes the most sense given what was transcribed and print the English translation as your most likely final translation. Transcribed text:  {model output}
"""

with gr.Blocks(css="style.css") as demo:
    gr.Markdown(INTRO_TEXT)
    with gr.Tab("Text Generation"):
        with gr.Column():
            image = gr.Image(type="pil")
            text_input = gr.Text(label="Input Text")

            text_output = gr.Text(label="Text Output")
            chat_btn = gr.Button()

        chat_inputs = [
            image,
            text_input
        ]
        chat_outputs = [
            text_output
        ]
        chat_btn.click(
            fn=infer,
            inputs=chat_inputs,
            outputs=chat_outputs,
        )

        examples = [["./diagnosis-1.png", "Transcribe the Arabic text."],
                    ["./sign.png", "Transcribe the Arabic text."]]
        gr.Markdown("")

        gr.Examples(
            examples=examples,
            inputs=chat_inputs,
        )

#########

if __name__ == "__main__":
    demo.queue(max_size=10).launch(debug=True)