- attn_norm=instancenorm, attn_projector=orthogonal, attn_weight=5, learning_rate=0.0001, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm, attn_projector=mlp, attn_weight=5, learning_rate=0.0001, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm, attn_projector=mlp, attn_weight=5, learning_rate=0.0002, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm, attn_projector=orthogonal, attn_weight=5, learning_rate=0.0001, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm, attn_projector=orthogonal, attn_weight=5, learning_rate=0.0002, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm_teacher_only, attn_projector=mlp, attn_weight=5, learning_rate=0.0001, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm_teacher_only, attn_projector=mlp, attn_weight=5, learning_rate=0.0002, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm_teacher_only, attn_projector=orthogonal, attn_weight=5, learning_rate=0.0001, per_device_train_batch_size=16, warmup_ratio=0
- attn_norm=layernorm_teacher_only, attn_projector=orthogonal, attn_weight=5, learning_rate=0.0002, per_device_train_batch_size=16, warmup_ratio=0