Update pipeline_stable_diffusion_3_ipa.py

pipeline_stable_diffusion_3_ipa.py

@@ -1147,31 +1147,47 @@ class StableDiffusion3Pipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingle
         if clip_image != None:
             print('Using primary image.')
             clip_image = clip_image.resize((max(clip_image.size), max(clip_image.size)))
-            clip_image_embeds_1 = self.encode_clip_image_emb(clip_image, device, dtype)
+            #clip_image_embeds_1 = self.encode_clip_image_emb(clip_image, device, dtype)
+            clip_image_embeds_1 = self.clip_image_processor(images=clip_image, return_tensors="pt").pixel_values
+            clip_image_embeds_1 = clip_image_embeds_1.to(device, dtype=dtype)
+            clip_image_embeds_1 = self.image_encoder(clip_image_embeds_1, output_hidden_states=True).hidden_states[-2]
+            clip_image_embeds_1 = clip_image_embeds_1 * scale_1
             image_prompt_embeds_list.append(clip_image_embeds_1)
         if clip_image_2 != None:
             print('Using secondary image.')
             clip_image_2 = clip_image_2.resize((max(clip_image_2.size), max(clip_image_2.size)))
-            image_prompt_embeds_2 = self.encode_clip_image_emb(clip_image_2, device, dtype)
-            image_prompt_embeds_2 = image_prompt_embeds_2 * scale_2
+            #clip_image_embeds_2 = self.encode_clip_image_emb(clip_image, device, dtype)
+            clip_image_embeds_2 = self.clip_image_processor(images=clip_image_2, return_tensors="pt").pixel_values
+            clip_image_embeds_2 = clip_image_embeds_2.to(device, dtype=dtype)
+            clip_image_embeds_2 = self.image_encoder(clip_image_embeds_2, output_hidden_states=True).hidden_states[-2]
+            clip_image_embeds_2 = clip_image_embeds_2 * scale_2
             image_prompt_embeds_list.append(image_prompt_embeds_2)
         if clip_image_3 != None:
             print('Using tertiary image.')
             clip_image_3 = clip_image_3.resize((max(clip_image_3.size), max(clip_image_3.size)))
-            image_prompt_embeds_3 = self.encode_clip_image_emb(clip_image_3, device, dtype)
-            image_prompt_embeds_3 = image_prompt_embeds_3 * scale_3
+            #clip_image_embeds_3 = self.encode_clip_image_emb(clip_image, device, dtype)
+            clip_image_embeds_3 = self.clip_image_processor(images=clip_image_3, return_tensors="pt").pixel_values
+            clip_image_embeds_3 = clip_image_embeds_3.to(device, dtype=dtype)
+            clip_image_embeds_3 = self.image_encoder(clip_image_embeds_3, output_hidden_states=True).hidden_states[-2]
+            clip_image_embeds_3 = clip_image_embeds_3 * scale_3
             image_prompt_embeds_list.append(image_prompt_embeds_3)
         if clip_image_4 != None:
             print('Using quaternary image.')
             clip_image_4 = clip_image_4.resize((max(clip_image_4.size), max(clip_image_4.size)))
-            image_prompt_embeds_4 = self.encode_clip_image_emb(clip_image_4, device, dtype)
-            image_prompt_embeds_4 = image_prompt_embeds_4 * scale_4
+            #clip_image_embeds_4 = self.encode_clip_image_emb(clip_image, device, dtype)
+            clip_image_embeds_4 = self.clip_image_processor(images=clip_image_4, return_tensors="pt").pixel_values
+            clip_image_embeds_4 = clip_image_embeds_4.to(device, dtype=dtype)
+            clip_image_embeds_2 = self.image_encoder(clip_image_embeds_4, output_hidden_states=True).hidden_states[-2]
+            clip_image_embeds_4 = clip_image_embeds_4 * scale_4
             image_prompt_embeds_list.append(image_prompt_embeds_4)
         if clip_image_5 != None:
             print('Using quinary image.')
             clip_image_5 = clip_image_5.resize((max(clip_image_5.size), max(clip_image_5.size)))
-            image_prompt_embeds_5 = self.encode_clip_image_emb(clip_image_5, device, dtype)
-            image_prompt_embeds_5 = image_prompt_embeds_5 * scale_5
+            #clip_image_embeds_5 = self.encode_clip_image_emb(clip_image, device, dtype)
+            clip_image_embeds_5 = self.clip_image_processor(images=clip_image_5, return_tensors="pt").pixel_values
+            clip_image_embeds_5 = clip_image_embeds_5.to(device, dtype=dtype)
+            clip_image_embeds_5 = self.image_encoder(clip_image_embeds_5, output_hidden_states=True).hidden_states[-2]
+            clip_image_embeds_5 = clip_image_embeds_5 * scale_5
             image_prompt_embeds_list.append(image_prompt_embeds_5)
             
         # Concatenate the image embeddings
@@ -1190,7 +1206,7 @@ class StableDiffusion3Pipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingle
             
         #clip_image_embeds = torch.mean(torch.stack(image_prompt_embeds_list), dim=0)  # working
             
-        clip_image_embeds = torch.cat(image_prompt_embeds_list).mean(dim=0).unsqueeze(0)
+        clip_image_embeds = torch.cat([torch.zeros_like(image_prompt_embeds_list),image_prompt_embeds_list]).mean(dim=0).unsqueeze(0)
         bs_embed, seq_len, _ = clip_image_embeds.shape
         clip_image_embeds = clip_image_embeds.repeat(1, 1, 1)
         clip_image_embeds = clip_image_embeds.view(2, -1)