import albumentations as A from albumentations import DualIAATransform, to_tuple import imgaug.augmenters as iaa import cv2 from tqdm import tqdm from sklearn.cluster import KMeans from sklearn.metrics import pairwise_distances_argmin from sklearn.utils import shuffle import numpy as np class IAAAffine2(DualIAATransform): """Place a regular grid of points on the input and randomly move the neighbourhood of these point around via affine transformations. Note: This class introduce interpolation artifacts to mask if it has values other than {0;1} Args: p (float): probability of applying the transform. Default: 0.5. Targets: image, mask """ def __init__( self, scale=(0.7, 1.3), translate_percent=None, translate_px=None, rotate=0.0, shear=(-0.1, 0.1), order=1, cval=0, mode="reflect", always_apply=False, p=0.5, ): super(IAAAffine2, self).__init__(always_apply, p) self.scale = dict(x=scale, y=scale) self.translate_percent = to_tuple(translate_percent, 0) self.translate_px = to_tuple(translate_px, 0) self.rotate = to_tuple(rotate) self.shear = dict(x=shear, y=shear) self.order = order self.cval = cval self.mode = mode @property def processor(self): return iaa.Affine( self.scale, self.translate_percent, self.translate_px, self.rotate, self.shear, self.order, self.cval, self.mode, ) def get_transform_init_args_names(self): return ("scale", "translate_percent", "translate_px", "rotate", "shear", "order", "cval", "mode") class IAAPerspective2(DualIAATransform): """Perform a random four point perspective transform of the input. Note: This class introduce interpolation artifacts to mask if it has values other than {0;1} Args: scale ((float, float): standard deviation of the normal distributions. These are used to sample the random distances of the subimage's corners from the full image's corners. Default: (0.05, 0.1). p (float): probability of applying the transform. Default: 0.5. Targets: image, mask """ def __init__(self, scale=(0.05, 0.1), keep_size=True, always_apply=False, p=0.5, order=1, cval=0, mode="replicate"): super(IAAPerspective2, self).__init__(always_apply, p) self.scale = to_tuple(scale, 1.0) self.keep_size = keep_size self.cval = cval self.mode = mode @property def processor(self): return iaa.PerspectiveTransform(self.scale, keep_size=self.keep_size, mode=self.mode, cval=self.cval) def get_transform_init_args_names(self): return ("scale", "keep_size") def get_bg_transforms(transform_variant, out_size): max_size = int(out_size * 1.2) if transform_variant == 'train': transform = [ A.SmallestMaxSize(max_size, always_apply=True, interpolation=cv2.INTER_AREA), A.RandomResizedCrop(out_size, out_size, scale=(0.9, 1.5), p=1, ratio=(0.9, 1.1)), ] else: transform = [ A.SmallestMaxSize(out_size, always_apply=True), A.RandomCrop(out_size, out_size, True), ] return A.Compose(transform) def get_fg_transforms(out_size, scale_limit=(-0.85, -0.3), transform_variant='train'): if transform_variant == 'train': transform = [ A.LongestMaxSize(out_size), A.RandomScale(scale_limit=scale_limit, always_apply=True, interpolation=cv2.INTER_AREA), IAAAffine2(scale=(1, 1), rotate=(-15, 15), shear=(-0.1, 0.1), p=0.3, mode='constant'), IAAPerspective2(scale=(0.0, 0.06), p=0.3, mode='constant'), A.HorizontalFlip(), A.ElasticTransform(alpha=0.3, sigma=15, alpha_affine=15, border_mode=cv2.BORDER_CONSTANT, p=0.3), A.GridDistortion(border_mode=cv2.BORDER_CONSTANT, p=0.3) ] elif transform_variant == 'distort_only': transform = [ IAAAffine2(scale=(1, 1), shear=(-0.1, 0.1), p=0.3, mode='constant'), IAAPerspective2(scale=(0.0, 0.06), p=0.3, mode='constant'), A.HorizontalFlip(), A.ElasticTransform(alpha=0.3, sigma=15, alpha_affine=15, border_mode=cv2.BORDER_CONSTANT, p=0.3), A.GridDistortion(border_mode=cv2.BORDER_CONSTANT, p=0.3) ] else: transform = [ A.LongestMaxSize(out_size), A.RandomScale(scale_limit=scale_limit, always_apply=True, interpolation=cv2.INTER_LINEAR) ] return A.Compose(transform) def get_transforms(transform_variant, out_size, to_float=True): if transform_variant == 'distortions': transform = [ IAAAffine2(scale=(1, 1.3), rotate=(-20, 20), shear=(-0.1, 0.1), p=1, mode='constant'), IAAPerspective2(scale=(0.0, 0.06), p=0.3, mode='constant'), A.OpticalDistortion(), A.HorizontalFlip(), A.Sharpen(p=0.3), A.CLAHE(), A.GaussNoise(p=0.3), A.Posterize(), A.ElasticTransform(alpha=0.3, sigma=15, alpha_affine=15, border_mode=cv2.BORDER_CONSTANT), ] elif transform_variant == 'default': transform = [ A.HorizontalFlip(), A.Rotate(20, p=0.3) ] elif transform_variant == 'identity': transform = [] else: raise ValueError(f'Unexpected transform_variant {transform_variant}') if to_float: transform.append(A.ToFloat()) return A.Compose(transform) def get_template_transforms(transform_variant, out_size, to_float=True): if transform_variant == 'distortions': transform = [ A.Cutout(p=0.3, max_w_size=30, max_h_size=30, num_holes=1), IAAAffine2(scale=(1, 1.3), rotate=(-20, 20), shear=(-0.1, 0.1), p=1, mode='constant'), IAAPerspective2(scale=(0.0, 0.06), p=0.3, mode='constant'), A.OpticalDistortion(), A.HorizontalFlip(), A.Sharpen(p=0.3), A.CLAHE(), A.GaussNoise(p=0.3), A.Posterize(), A.ElasticTransform(alpha=0.3, sigma=15, alpha_affine=15, border_mode=cv2.BORDER_CONSTANT), ] elif transform_variant == 'identity': transform = [] else: raise ValueError(f'Unexpected transform_variant {transform_variant}') if to_float: transform.append(A.ToFloat()) return A.Compose(transform) def rotate_image(mat: np.ndarray, angle: float, alpha_crop: bool = False) -> np.ndarray: """ Rotates an image (angle in degrees) and expands image to avoid cropping # https://stackoverflow.com/questions/43892506/opencv-python-rotate-image-without-cropping-sides """ height, width = mat.shape[:2] # image shape has 3 dimensions image_center = (width/2, height/2) # getRotationMatrix2D needs coordinates in reverse order (width, height) compared to shape rotation_mat = cv2.getRotationMatrix2D(image_center, angle, 1.) # rotation calculates the cos and sin, taking absolutes of those. abs_cos = abs(rotation_mat[0,0]) abs_sin = abs(rotation_mat[0,1]) # find the new width and height bounds bound_w = int(height * abs_sin + width * abs_cos) bound_h = int(height * abs_cos + width * abs_sin) # subtract old image center (bringing image back to origo) and adding the new image center coordinates rotation_mat[0, 2] += bound_w/2 - image_center[0] rotation_mat[1, 2] += bound_h/2 - image_center[1] # rotate image with the new bounds and translated rotation matrix rotated_mat = cv2.warpAffine(mat, rotation_mat, (bound_w, bound_h)) if alpha_crop and len(rotated_mat.shape) == 3 and rotated_mat.shape[-1] == 4: x, y, w, h = cv2.boundingRect(rotated_mat[..., -1]) rotated_mat = rotated_mat[y: y+h, x: x+w] return rotated_mat def recreate_image(codebook, labels, w, h): """Recreate the (compressed) image from the code book & labels""" return (codebook[labels].reshape(w, h, -1) * 255).astype(np.uint8) def quantize_image(image: np.ndarray, n_colors: int, method='kmeans', mask=None): # https://scikit-learn.org/stable/auto_examples/cluster/plot_color_quantization.html image = np.array(image, dtype=np.float64) / 255 if len(image.shape) == 3: w, h, d = tuple(image.shape) else: w, h = image.shape d = 1 # assert d == 3 image_array = image.reshape(-1, d) if method == 'kmeans': image_array_sample = None if mask is not None: ids = np.where(mask) if len(ids[0]) > 10: bg = image[ids][::2] fg = image[np.where(mask == 0)] max_bg_num = int(fg.shape[0] * 1.5) if bg.shape[0] > max_bg_num: bg = shuffle(bg, random_state=0, n_samples=max_bg_num) image_array_sample = np.concatenate((fg, bg), axis=0) if image_array_sample.shape[0] > 2048: image_array_sample = shuffle(image_array_sample, random_state=0, n_samples=2048) else: image_array_sample = None if image_array_sample is None: image_array_sample = shuffle(image_array, random_state=0, n_samples=2048) kmeans = KMeans(n_clusters=n_colors, n_init=10, random_state=0).fit( image_array_sample ) labels = kmeans.predict(image_array) quantized = recreate_image(kmeans.cluster_centers_, labels, w, h) return quantized, kmeans.cluster_centers_, labels else: codebook_random = shuffle(image_array, random_state=0, n_samples=n_colors) labels_random = pairwise_distances_argmin(codebook_random, image_array, axis=0) return [recreate_image(codebook_random, labels_random, w, h)] def resize2height(img: np.ndarray, height: int): im_h, im_w = img.shape[:2] if im_h > height: interpolation = cv2.INTER_AREA else: interpolation = cv2.INTER_LINEAR if im_h != height: img = cv2.resize(img, (int(height / im_h * im_w), height), interpolation=interpolation) return img if __name__ == '__main__': import os.path as osp img_path = r'tmp\megumin.png' save_dir = r'tmp' sample_num = 24 tv = 'distortions' out_size = 224 transforms = get_transforms(tv, out_size ,to_float=False) img = cv2.imread(img_path) for idx in tqdm(range(sample_num)): transformed = transforms(image=img)['image'] print(transformed.shape) cv2.imwrite(osp.join(save_dir, str(idx)+'-transform.jpg'), transformed) # cv2.waitKey(0) pass