Fix render

Image3DProcessor.py

@@ -86,7 +86,7 @@ class Image3DProcessor:
         world_view_transforms, full_proj_transforms, camera_centers = get_target_cameras()
         background = torch.tensor([1, 1, 1], dtype=torch.float32, device=self.device)
         loop_renders = []
-        t_to_512 = torchvision.transforms.Resize(512, interpolation=torchvision.transforms.InterpolationMode.NEAREST)
+        t_to_512 = torchvision.transforms.Resize(512, interpolation=torchvision.transforms.InterpolationMode.LANCZOS)
 
         for r_idx in range(world_view_transforms.shape[0]):
             rendered_image = render_predicted(
@@ -101,16 +101,16 @@ class Image3DProcessor:
             rendered_image = t_to_512(rendered_image)
             loop_renders.append(torch.clamp(rendered_image * 255, 0.0, 255.0).detach().permute(1, 2, 0).cpu().numpy().astype(np.uint8))
 
-        # Save the mesh to an in-memory buffer
-        mesh_buffer = BytesIO()
-        export_to_obj(mesh_buffer, reconstruction)  # Ensure `export_to_obj()` can write to `BytesIO`
-        mesh_buffer.seek(0)  # Reset buffer position for reading
-        mesh_data = mesh_buffer.getvalue()
+        # Save video to a file and load its content
+        video_path = "loop_.mp4"
+        imageio.mimsave(video_path, loop_renders, fps=25)
+        with open(video_path, "rb") as video_file:
+            video_data = video_file.read()
 
-        # Save video to an in-memory buffer
-        video_buffer = BytesIO()
-        imageio.mimwrite(video_buffer, loop_renders, format='mp4', fps=30)
-        video_buffer.seek(0)  # Reset buffer position for reading
-        video_data = video_buffer.getvalue()
+        # Save the mesh as a .ply file and load its content
+        mesh_path = "mesh.ply"
+        export_to_obj(reconstruction, mesh_path)
+        with open(mesh_path, "rb") as mesh_file:
+            mesh_data = mesh_file.read()
 
-        return mesh_data, video_data
+        return mesh_data, video_data

app.py

@@ -18,9 +18,7 @@ class VideoGenerator:
 
     def generate_3d_video(self, image):
         # Process the image and create a 3D video and mesh
-        processed_image = self.processor.preprocess(image, preprocess_background=False)
-        mesh_data, video_data = self.processor.reconstruct_and_export(processed_image)
-
+        mesh_data, video_data = self.processor.reconstruct_and_export(image)
         return mesh_data, video_data
 
 class GradioApp:
@@ -36,18 +34,19 @@ class GradioApp:
 
     def full_pipeline(self, prompt):
         initial_image = self.sdxl_generator.generate_images([prompt])[0]
+        # controlled_image = self.controlnet_processor.controlnet_image(initial_image)
         mesh_data, video_data = self.video_generator.generate_3d_video(initial_image)
         
         # Create temporary files to display mesh and video content
-        #with tempfile.NamedTemporaryFile(delete=False, suffix=".obj") as mesh_file:
-            #mesh_file.write(mesh_data)
-            #mesh_path = mesh_file.name
+        with tempfile.NamedTemporaryFile(delete=False, suffix=".ply") as mesh_file:
+            mesh_file.write(mesh_data)
+            mesh_path = mesh_file.name
         
         with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as video_file:
             video_file.write(video_data)
             video_path = video_file.name
         
-        return initial_image, video_path
+        return initial_image, mesh_path, video_path
 
     def launch(self):
         interface = gr.Interface(
@@ -55,13 +54,13 @@ class GradioApp:
             inputs=gr.Textbox(label="Input Prompt"),
             outputs=[
                 gr.Image(label="Generated Image"),
-                #gr.File(label="3D Mesh (.obj)"),
+                gr.File(label="3D Mesh (.ply)"),
                 gr.Video(label="3D Model Video")
             ],
             title="SDXL to ControlNet to 3D Pipeline",
             description="Generate an image using SDXL, refine it with ControlNet, and generate a 3D video output."
         )
-        interface.launch(share=True)
+        interface.launch(share=True)  # Added `share=True` for public link
 
 if __name__ == "__main__":
     app = GradioApp()