from collections import defaultdict import gradio import numpy as np import torch import cv2 from PIL import Image from dp2 import utils from tops.config import instantiate import tops import gradio.inputs from stylemc import get_and_cache_direction, get_styles from sg3_torch_utils.ops import grid_sample_gradfix, bias_act, upfirdn2d grid_sample_gradfix.enabled = False bias_act.enabled = False upfirdn2d.enabled = False class GuidedDemo: def __init__(self, face_anonymizer, cfg_face, multi_modal_truncation, truncation_value) -> None: self.anonymizer = face_anonymizer self.multi_modal_truncation = multi_modal_truncation self.truncation_value = truncation_value assert sum([x is not None for x in list(face_anonymizer.generators.values())]) == 1 self.generator = [x for x in list(face_anonymizer.generators.values()) if x is not None][0] face_G_cfg = utils.load_config(cfg_face.anonymizer.face_G_cfg) face_G_cfg.train.batch_size = 1 self.dl = instantiate(face_G_cfg.data.val.loader) self.cache_dir = face_G_cfg.output_dir self.precompute_edits() def precompute_edits(self): self.precomputed_edits = set() for edit in self.precomputed_edits: get_and_cache_direction(self.cache_dir, self.dl, self.generator, edit) if self.cache_dir.joinpath("stylemc_cache").is_dir(): for path in self.cache_dir.joinpath("stylemc_cache").iterdir(): text_prompt = path.stem.replace("_", " ") self.precomputed_edits.add(text_prompt) print(text_prompt) self.edits = defaultdict(defaultdict) def anonymize(self, img, show_boxes: bool, current_box_idx: int, current_styles, current_boxes, update_identity, edits, cache_id=None): if not isinstance(img, torch.Tensor): img, cache_id = pil2torch(img) img = tops.to_cuda(img) current_box_idx = current_box_idx % len(current_boxes) edited_styles = [s.clone() for s in current_styles] for face_idx, face_edits in edits.items(): for prompt, strength in face_edits.items(): direction = get_and_cache_direction(self.cache_dir, self.dl, self.generator, prompt) edited_styles[int(face_idx)] += direction * strength update_identity[int(face_idx)] = True assert img.dtype == torch.uint8 img = self.anonymizer( img, truncation_value=self.truncation_value, multi_modal_truncation=self.multi_modal_truncation, amp=True, cache_id=cache_id, all_styles=edited_styles, update_identity=update_identity) update_identity = [True for i in range(len(update_identity))] img = utils.im2numpy(img) if show_boxes: x0, y0, x1, y1 = [int(_) for _ in current_boxes[int(current_box_idx)]] img = cv2.rectangle(img, (x0, y0), (x1, y1), (255, 0, 0), 1) return img, update_identity def update_image(self, img, show_boxes): img, cache_id = pil2torch(img) img = tops.to_cuda(img) det = self.anonymizer.detector.forward_and_cache(img, cache_id, load_cache=True)[0] current_styles = [] for i in range(len(det)): s = get_styles( np.random.randint(0, 999999), self.generator, None, truncation_value=self.truncation_value) current_styles.append(s) update_identity = [True for i in range(len(det))] current_boxes = np.array(det.boxes) edits = defaultdict(defaultdict) cur_face_idx = -1 % len(current_boxes) img, update_identity = self.anonymize( img, show_boxes, cur_face_idx, current_styles, current_boxes, update_identity, edits, cache_id=cache_id) return img, current_styles, current_boxes, update_identity, edits, cur_face_idx def change_face(self, change, cur_face_idx, current_boxes, input_image, show_boxes, current_styles, update_identity, edits): cur_face_idx = (cur_face_idx + change) % len(current_boxes) img, update_identity = self.anonymize( input_image, show_boxes, cur_face_idx, current_styles, current_boxes, update_identity, edits) return img, update_identity, cur_face_idx def add_style(self, face_idx: int, prompt: str, strength: float, input_image, show_boxes, current_styles, current_boxes, update_identity, edits): face_idx = face_idx % len(current_boxes) edits[face_idx][prompt] = strength img, update_identity = self.anonymize( input_image, show_boxes, face_idx, current_styles, current_boxes, update_identity, edits) return img, update_identity, edits def setup_interface(self): current_styles = gradio.State() current_boxes = gradio.State(None) update_identity = gradio.State([]) edits = gradio.State([]) with gradio.Row(): input_image = gradio.Image( type="pil", label="Upload your image or try the example below!", source="webcam") output_image = gradio.Image(type="numpy", label="Output") with gradio.Row(): update_btn = gradio.Button("Update Anonymization").style(full_width=True) with gradio.Row(): show_boxes = gradio.Checkbox(value=True, label="Show Selected") cur_face_idx = gradio.Number(value=-1, label="Current", interactive=False) previous = gradio.Button("Previous Person") next_ = gradio.Button("Next Person") with gradio.Row(): text_prompt = gradio.Textbox( placeholder=" | ".join(list(self.precomputed_edits)), label="Text Prompt for Edit") edit_strength = gradio.Slider(0, 5, step=.01) add_btn = gradio.Button("Add Edit") add_btn.click( self.add_style, inputs=[cur_face_idx, text_prompt, edit_strength, input_image, show_boxes,current_styles, current_boxes, update_identity, edits], outputs=[output_image, update_identity, edits]) update_btn.click( self.update_image, inputs=[input_image, show_boxes], outputs=[output_image, current_styles, current_boxes, update_identity, edits, cur_face_idx]) input_image.change( self.update_image, inputs=[input_image, show_boxes], outputs=[output_image, current_styles, current_boxes, update_identity, edits, cur_face_idx]) previous.click( self.change_face, inputs=[gradio.State(-1), cur_face_idx, current_boxes, input_image, show_boxes, current_styles, update_identity, edits], outputs=[output_image, update_identity, cur_face_idx]) next_.click( self.change_face, inputs=[gradio.State(1), cur_face_idx, current_boxes, input_image, show_boxes,current_styles, update_identity, edits], outputs=[output_image, update_identity, cur_face_idx]) show_boxes.change( self.anonymize, inputs=[input_image, show_boxes, cur_face_idx, current_styles, current_boxes, update_identity, edits], outputs=[output_image, update_identity]) class WebcamDemo: def __init__(self, anonymizer) -> None: self.anonymizer = anonymizer with gradio.Row(): input_image = gradio.Image(type="pil", source="webcam", streaming=True) output_image = gradio.Image(type="numpy", label="Output") with gradio.Row(): truncation_value = gradio.Slider(0, 1, value=0, step=0.01) truncation = gradio.Radio(["Multi-modal truncation", "Unimodal truncation"], value="Unimodal truncation") with gradio.Row(): visualize_det = gradio.Checkbox(value=False, label="Show Detections") track = gradio.Checkbox(value=False, label="Track detections (samples same latent variable per track)") input_image.stream( self.anonymize, inputs=[input_image, visualize_det, truncation_value,truncation, track, gradio.Variable(False)], outputs=[output_image]) self.track = True def anonymize(self, img: Image, visualize_detection: bool, truncation_value, truncation_type, track, reset_track): if reset_track: self.anonymizer.reset_tracker() mmt = truncation_type == "Multi-modal truncation" img, cache_id = pil2torch(img) img = tops.to_cuda(img) self.anonymizer if visualize_detection: img = self.anonymizer.visualize_detection(img, cache_id=cache_id) else: img = self.anonymizer( img, truncation_value=truncation_value, multi_modal_truncation=mmt, amp=True, cache_id=cache_id, track=track) img = utils.im2numpy(img) return img class ExampleDemo(WebcamDemo): def __init__(self, anonymizer, source="webcam") -> None: self.anonymizer = anonymizer kwargs = dict(source=source) if source is None: kwargs.pop("source") with gradio.Row(): input_image = gradio.Image(type="pil", **kwargs) output_image = gradio.Image(type="numpy", label="Output") with gradio.Row(): update_btn = gradio.Button("Update Anonymization").style(full_width=True) resample = gradio.Button("Resample Latent Variables").style(full_width=True) with gradio.Row(): truncation_value = gradio.Slider(0, 1, value=0, step=0.01) truncation = gradio.Radio(["Multi-modal truncation", "Unimodal truncation"], value="Unimodal truncation") visualize_det = gradio.Checkbox(value=False, label="Show Detections") visualize_det.change( self.anonymize, inputs=[input_image, visualize_det, truncation_value, truncation, gradio.Variable(True), gradio.Variable(False)], outputs=[output_image]) gradio.Examples( ["media/erling.jpg", "media/regjeringen.jpg"], inputs=[input_image] ) update_btn.click( self.anonymize, inputs=[input_image, visualize_det, truncation_value, truncation, gradio.Variable(True), gradio.Variable(False)], outputs=[output_image]) resample.click( self.anonymize, inputs=[input_image, visualize_det, truncation_value, truncation, gradio.Variable(True), gradio.Variable(True)], outputs=[output_image]) input_image.change( self.anonymize, inputs=[input_image, visualize_det, truncation_value, truncation, gradio.Variable(False), gradio.Variable(True)], outputs=[output_image]) self.track = False self.truncation_value = truncation_value class Information: def __init__(self) -> None: gradio.Markdown("##