| 05/25/2023 17:40:41 - INFO - __main__ - ***** Running training ***** | |
| 05/25/2023 17:40:41 - INFO - __main__ - Num examples = 833 | |
| 05/25/2023 17:40:41 - INFO - __main__ - Num Epochs = 72 | |
| 05/25/2023 17:40:41 - INFO - __main__ - Instantaneous batch size per device = 1 | |
| 05/25/2023 17:40:41 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 4 | |
| 05/25/2023 17:40:41 - INFO - __main__ - Gradient Accumulation steps = 4 | |
| 05/25/2023 17:40:41 - INFO - __main__ - Total optimization steps = 15000 | |
| Steps: 0%| | 0/15000 [00:00<?, ?it/s]Traceback (most recent call last): | |
| File "lora_test_1.py", line 908, in <module> | |
| main() | |
| File "lora_test_1.py", line 728, in main | |
| latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample() | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/diffusers/utils/accelerate_utils.py", line 46, in wrapper | |
| return method(self, *args, **kwargs) | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/diffusers/models/autoencoder_kl.py", line 164, in encode | |
| h = self.encoder(x) | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/diffusers/models/vae.py", line 109, in forward | |
| sample = self.conv_in(sample) | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl | |
| return forward_call(*args, **kwargs) | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 463, in forward | |
| return self._conv_forward(input, self.weight, self.bias) | |
| File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 459, in _conv_forward | |
| return F.conv2d(input, weight, bias, self.stride, | |
| RuntimeError: cuDNN error: CUDNN_STATUS_NOT_INITIALIZED |