File size: 2,121 Bytes
e58ac13 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 |
05/25/2023 17:53:55 - INFO - __main__ - ***** Running training *****
05/25/2023 17:53:55 - INFO - __main__ - Num examples = 833
05/25/2023 17:53:55 - INFO - __main__ - Num Epochs = 72
05/25/2023 17:53:55 - INFO - __main__ - Instantaneous batch size per device = 1
05/25/2023 17:53:55 - INFO - __main__ - Total train batch size (w. parallel, distributed & accumulation) = 4
05/25/2023 17:53:55 - INFO - __main__ - Gradient Accumulation steps = 4
05/25/2023 17:53:55 - INFO - __main__ - Total optimization steps = 15000
Steps: 0%| | 0/15000 [00:00<?, ?it/s]Traceback (most recent call last):
File "lora_test_1.py", line 908, in <module>
main()
File "lora_test_1.py", line 728, in main
latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample()
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/diffusers/utils/accelerate_utils.py", line 46, in wrapper
return method(self, *args, **kwargs)
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/diffusers/models/autoencoder_kl.py", line 164, in encode
h = self.encoder(x)
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/diffusers/models/vae.py", line 109, in forward
sample = self.conv_in(sample)
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/module.py", line 1501, in _call_impl
return forward_call(*args, **kwargs)
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 463, in forward
return self._conv_forward(input, self.weight, self.bias)
File "/home/long.qul/miniconda3/envs/triton/lib/python3.8/site-packages/torch/nn/modules/conv.py", line 459, in _conv_forward
return F.conv2d(input, weight, bias, self.stride,
RuntimeError: cuDNN error: CUDNN_STATUS_NOT_INITIALIZED |