First model version

app.py

@@ -1,18 +1,29 @@
 import os
 os.system('pip install --upgrade --no-cache-dir gdown')
 os.system('gdown -O ./output/ctw/model_ctw.pth 1Ajslu_9WisuZ2nJGzE6qbD87aK6_ozzA')
+os.system('gdown -O ./workdir.zip 1mYM_26qHUom_5NU7iutHneB_KHlLjL5y')
+os.system('unzip workdir.zip')
 os.system('pip install "git+https://github.com/philferriere/cocoapi.git#egg=pycocotools&subdirectory=PythonAPI"')
 os.system('python setup.py build develop --user')
-#os.system('pip install git+https://github.com/Cyril-Sterling/ContourNet')
 
 import cv2
 import pandas as pd
 import gradio as gr
+
 from det_demo import DetDemo
 from maskrcnn_benchmark.config import cfg
 
+from demo import get_model, preprocess, postprocess, load
+from utils import Config, Logger, CharsetMapper
+
 
 def infer(filepath):
+    config = Config('configs/rec/train_abinet.yaml')
+    config.model_vision_checkpoint = None
+    model = get_model(config)
+    model = load(model, 'workdir/train-abinet/best-train-abinet.pth')
+    charset = CharsetMapper(filename=config.dataset_charset_path, max_length=config.dataset_max_length + 1)
+    
     cfg.merge_from_file('./configs/det/r50_baseline.yaml')
     # manual override some options
     cfg.merge_from_list(["MODEL.DEVICE", "cpu"])
@@ -23,9 +34,15 @@ def infer(filepath):
         confidence_threshold=0.7,
         output_polygon=True
     )
+    
     image = cv2.imread(filepath)
+    patchs = [image[box[1]:box[3], box[0]:box[2], :] for box in result_boxes]
+    patchs = [preprocess(patch, config.dataset_image_width, config.dataset_image_height) for patch in patchs]
+    patchs = torch.stack(patchs, dim=0)
+    
     result_polygons, result_masks, result_boxes = det_demo.run_on_opencv_image(image)
     visual_image = det_demo.visualization(image.copy(), result_polygons, result_masks, result_boxes)
+    
     cv2.imwrite('result.jpg', visual_image)
     return 'result.jpg'#, pd.DataFrame(result_words)
 

configs/ic/r50_baseline.yaml

@@ -1,75 +0,0 @@
-OUTPUT_DIR: "./output/ic15"
-MODEL:
-  META_ARCHITECTURE: "GeneralizedRCNN"
-  WEIGHT: catalog://ImageNetPretrained/MSRA/R-50
-  BACKBONE:
-    CONV_BODY: "R-50-FPN"
-  RESNETS:
-    BACKBONE_OUT_CHANNELS: 256
-  RPN:
-    USE_FPN: True
-    ANCHOR_STRIDE: (4, 8, 16, 32, 64)
-    ASPECT_RATIOS: (0.25, 0.5, 1.0, 2.0, 4.0)
-  ROI_HEADS:
-    USE_FPN: True
-    SCORE_THRESH: 0.52  # ic15
-    NMS: 0.89
-  ROI_BOX_HEAD:
-    DEFORMABLE_POOLING: False
-    POOLER_RESOLUTION: 7
-    POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125)
-    POOLER_SAMPLING_RATIO: 2
-    FEATURE_EXTRACTOR: "FPN2MLPFeatureExtractor"
-    PREDICTOR: "FPNPredictor"
-    NUM_CLASSES: 2
-    CLASS_WEIGHT: 1.0
-  ## Boundary
-  BOUNDARY_ON: True
-  ROI_BOUNDARY_HEAD:
-    DEFORMABLE_POOLING: False
-    FEATURE_EXTRACTOR: "BoundaryRCNNFPNFeatureExtractor"
-    POOLER_RESOLUTION: 14
-    POOLER_SCALES: (0.25, 0.125, 0.0625, 0.03125)
-    POOLER_SAMPLING_RATIO: 2
-    PREDICTOR: "BoundaryRCNNC4Predictor"
-    RESOLUTION: 48
-    SHARE_BOX_FEATURE_EXTRACTOR: False
-    BO_WEIGHT: 0.1
-    Loss_balance: 1.0
-
-PROCESS:
-  PNMS: True
-  NMS_THRESH: 0.25
-DATASETS:
-  TRAIN: ("ic15_train",)
-  TEST: ("ic15_test",)
-  Test_Visual: True
-DATALOADER:
-  SIZE_DIVISIBILITY: 32
-SOLVER:
-  BASE_LR: 0.00025
-  BIAS_LR_FACTOR: 2
-  WEIGHT_DECAY: 0.0001
-#  STEPS: (120000, 160000)
-  STEPS: (5000, 10000)  # fine-tune
-#  MAX_ITER: 180000
-  MAX_ITER: 190500  # fine-tune
-  IMS_PER_BATCH: 1
-  CHECKPOINT_PERIOD: 5000
-INPUT:
-
-  MIN_SIZE_TRAIN: (400,600,720,1000,1200)
-  MAX_SIZE_TRAIN: 2000
-  MIN_SIZE_TEST: 1200
-  MAX_SIZE_TEST: 2000
-
-  CROP_PROB_TRAIN: 1.0
-  ROTATE_PROB_TRAIN: 0.3  # fine-tune
-#  ROTATE_PROB_TRAIN: 1.0
-#  ROTATE_DEGREE: (0,30,60,90,210,150,180,210,240,270,300,330,360)
-  ROTATE_DEGREE: (10,)  # fine-tune
-
-TEST:
-  IMS_PER_BATCH: 1
-
-

configs/rec/template.yaml

@@ -0,0 +1,67 @@
+global:
+  name: exp
+  phase: train
+  stage: pretrain-vision
+  workdir: /tmp/workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 128
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 128
+  }
+  charset_path: data/charset_36.txt
+  num_workers: 4
+  max_length: 25  # 30
+  image_height: 32
+  image_width: 128
+  case_sensitive: False
+  eval_case_sensitive: False
+  data_aug: True
+  multiscales: False
+  pin_memory: True
+  smooth_label: False
+  smooth_factor: 0.1
+  one_hot_y: True
+  use_sm: False
+
+training:
+  epochs: 6
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 20000
+  start_iters: 0
+  stats_iters: 100000
+
+optimizer:
+  type: Adadelta # Adadelta, Adam
+  true_wd: False
+  wd: 0. # 0.001
+  bn_wd: False
+  args: {
+    # betas: !!python/tuple [0.9, 0.99], # betas=(0.9,0.99) for AdamW
+    # betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  clip_grad: 20
+  lr: [1.0, 1.0, 1.0]  # lr: [0.005, 0.005, 0.005]   
+  scheduler: {
+    periods: [3, 2, 1],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_abinet.ABINetModel'
+  checkpoint: ~
+  strict: True

configs/rec/train_abinet.yaml

@@ -0,0 +1,71 @@
+global:
+  name: train-abinet
+  phase: train
+  stage: train-super
+  workdir: workdir
+  seed: ~
+ 
+dataset:
+  train: {
+    roots: ['data/training/MJ/MJ_train/', 
+            'data/training/MJ/MJ_test/', 
+            'data/training/MJ/MJ_valid/', 
+            'data/training/ST'],
+    batch_size: 384
+  }
+  test: {
+    roots: ['data/evaluation/IIIT5k_3000', 
+            'data/evaluation/SVT', 
+            'data/evaluation/SVTP',
+            'data/evaluation/IC13_857',
+            'data/evaluation/IC15_1811',
+            'data/evaluation/CUTE80'],
+    batch_size: 384
+  }
+  data_aug: True
+  multiscales: False
+  num_workers: 14
+
+training:
+  epochs: 10
+  show_iters: 50
+  eval_iters: 3000
+  save_iters: 3000
+
+optimizer:
+  type: Adam
+  true_wd: False
+  wd: 0.0
+  bn_wd: False
+  clip_grad: 20
+  lr: 0.0001
+  args: {
+    betas: !!python/tuple [0.9, 0.999], # for default Adam 
+  }
+  scheduler: {
+    periods: [6, 4],
+    gamma: 0.1,
+  }
+
+model:
+  name: 'modules.model_abinet_iter.ABINetIterModel'
+  iter_size: 3
+  ensemble: ''
+  use_vision: False
+  vision: {
+    checkpoint: workdir/pretrain-vision-model/best-pretrain-vision-model.pth,
+    loss_weight: 1.,
+    attention: 'position',
+    backbone: 'transformer',
+    backbone_ln: 3,
+  }
+  language: {
+    checkpoint:  workdir/pretrain-language-model/pretrain-language-model.pth,
+    num_layers: 4,
+    loss_weight: 1.,
+    detach: True,
+    use_self_attn: False
+  }
+  alignment: {
+    loss_weight: 1.,
+  }