import os import gradio as gr from PIL import Image import torch from transformers import ViTForImageClassification, ViTImageProcessor from datasets import load_dataset, DownloadConfig import matplotlib.pyplot as plt import numpy as np import cv2 import requests # Mistral AI API configuration MISTRAL_API_KEY = "eoiBrPQzLjwNgOFgD7I4A4XF3TJOgBet" MISTRAL_API_URL = "https://api.mistral.ai/v1/chat/completions" def get_mistral_completion(prompt): headers = { "Content-Type": "application/json", "Authorization": f"Bearer {MISTRAL_API_KEY}" } data = { "model": "mistral-medium", "messages": [{"role": "user", "content": prompt}], "temperature": 0.7 } response = requests.post(MISTRAL_API_URL, headers=headers, json=data) if response.status_code == 200: return response.json()['choices'][0]['message']['content'] else: return "Error getting AI explanation. Please try again." # Model and processor configuration model_name_or_path = "google/vit-base-patch16-224-in21k" processor = ViTImageProcessor.from_pretrained(model_name_or_path) # Load dataset dataset_path = "pawlo2013/chest_xray" download_config = DownloadConfig(max_retries=10) train_dataset = load_dataset(dataset_path, split="train", download_config=download_config) class_names = train_dataset.features["label"].names # Load ViT model model = ViTForImageClassification.from_pretrained( "./models", num_labels=len(class_names), id2label={str(i): label for i, label in enumerate(class_names)}, label2id={label: i for i, label in enumerate(class_names)}, ) model.eval() def get_ai_explanation(diagnosis, probabilities): if diagnosis == "normal": prompt = f"""Given a chest X-ray analysis showing NORMAL results with {probabilities['normal']:.2%} confidence: 1. Explain what this means, please remember that NORMAL ['normal'] means this user does not have any Pneumonia. 2. Suggest when they should still consider consulting a doctor even though this user does not have any penumonia as per the test result. 3. List key symptoms that would warrant medical attention. Always identify yourself as PneumoInsight Bot. Keep the tone professional yet reassuring.""" else: prompt = f"""Given a chest X-ray analysis showing {diagnosis} pneumonia with {probabilities[diagnosis]:.2%} confidence: 1. Explain what {diagnosis} pneumonia is . Always identify yourself as PneumoInsight Bot. 2. List immediate steps the patient should take 3. Provide care recommendations 4. Mention warning signs to watch for Keep the tone informative and caring but emphasize the importance of professional medical consultation.""" return get_mistral_completion(prompt) def classify_and_visualize(img, device="cpu", discard_ratio=0.9, head_fusion="mean"): img = img.convert("RGB") processed_input = processor(images=img, return_tensors="pt").to(device) processed_input = processed_input["pixel_values"].to(device) with torch.no_grad(): outputs = model(processed_input, output_attentions=True) logits = outputs.logits probabilities = torch.softmax(logits, dim=1)[0].tolist() prediction = torch.argmax(logits, dim=-1).item() predicted_class = class_names[prediction] result = {class_name: prob for class_name, prob in zip(class_names, probabilities)} # Generate attention heatmap heatmap_img = show_final_layer_attention_maps( outputs, processed_input, device, discard_ratio, head_fusion ) return {"probabilities": result, "heatmap": heatmap_img} def show_final_layer_attention_maps( outputs, processed_input, device, discard_ratio=0.6, head_fusion="max", only_last_layer=False, ): with torch.no_grad(): image = processed_input.squeeze(0) image = image - image.min() image = image / image.max() result = torch.eye(outputs.attentions[0].size(-1)).to(device) if only_last_layer: attention_list = outputs.attentions[-1].unsqueeze(0).to(device) else: attention_list = outputs.attentions for attention in attention_list: if head_fusion == "mean": attention_heads_fused = attention.mean(axis=1) elif head_fusion == "max": attention_heads_fused = attention.max(axis=1)[0] elif head_fusion == "min": attention_heads_fused = attention.min(axis=1)[0] flat = attention_heads_fused.view(attention_heads_fused.size(0), -1) _, indices = flat.topk(int(flat.size(-1) * discard_ratio), -1, False) indices = indices[indices != 0] flat[0, indices] = 0 I = torch.eye(attention_heads_fused.size(-1)).to(device) a = (attention_heads_fused + 1.0 * I) / 2 a = a / a.sum(dim=-1) result = torch.matmul(a, result) mask = result[0, 0, 1:] width = int(mask.size(-1) ** 0.5) mask = mask.reshape(width, width).cpu().numpy() mask = mask / np.max(mask) mask = cv2.resize(mask, (224, 224)) mask = (mask - np.min(mask)) / (np.max(mask) - np.min(mask)) heatmap = plt.cm.jet(mask)[:, :, :3] showed_img = image.permute(1, 2, 0).detach().cpu().numpy() showed_img = (showed_img - np.min(showed_img)) / ( np.max(showed_img) - np.min(showed_img) ) superimposed_img = heatmap * 0.4 + showed_img * 0.6 superimposed_img_pil = Image.fromarray( (superimposed_img * 255).astype(np.uint8) ) return superimposed_img_pil def load_examples_from_folder(folder_path): examples = [] if os.path.exists(folder_path): for file in os.listdir(folder_path): if file.endswith((".png", ".jpg", ".jpeg")): examples.append(os.path.join(folder_path, file)) return examples def create_interface(): # Custom CSS custom_css = """ .logo-container { text-align: center; margin-bottom: 20px; } .logo-container img { max-width: 300px; } .welcome-message { text-align: center; margin: 20px 0; padding: 20px; background-color: #f5f5f5; border-radius: 10px; } .model-explanation { margin: 20px 0; padding: 20px; background-color: #f0f7ff; border-radius: 10px; } .pneumonia-info { margin: 20px 0; padding: 20px; background-color: #fff5f5; border-radius: 10px; } .disclaimer { margin-top: 20px; padding: 20px; background-color: #f5f5f5; border-radius: 10px; font-size: 0.9em; } """ # HTML Components logo_html = """
Our system uses a Vision Transformer (ViT) model to analyze chest X-ray images. The attention heatmap visualizes areas the AI focuses on while making its diagnosis, helping make the decision-making process more transparent. The warmer colors (red/yellow) indicate areas of higher attention.
Credits: The attention heatmap visualization is implemented using the attention rollout technique by jacobgil.
Pneumonia is an infection that inflames the air sacs in one or both lungs. Common symptoms include:
Prevention tips:
This tool is for educational purposes only and should not be used as a substitute for professional medical advice, diagnosis, or treatment. Always seek the advice of your physician or other qualified health provider.
Created by the team at VIT-AP University. View the source code on GitHub.