Ashwin V. Mohanan commited on
Commit
97ba84c
·
1 Parent(s): 9714711

Digitize and visualize

Browse files
app/assets/jinja-templates/image.j2 ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ <svg viewBox="0 0 {{ page.width }} {{ page.height }}" xmlns="http://www.w3.org/2000/svg">
2
+ <image height="{{ page.height }}" width="{{ page.width }}" href="/gradio_api/file={{ page.path }}" />
3
+ {%- for line in page.cells -%}
4
+ <a class="cellline line{{loop.index}} highlighted" onmouseover="document.querySelectorAll('.line{{loop.index}}').forEach(element => {element.classList.remove('highlighted')});" onmouseout="document.querySelectorAll('.line{{loop.index}}').forEach(element => {element.classList.add('highlighted')});">
5
+ <polygon id="{{ loop.index }}" points="{% for point in line.polygon %}{{ point|join(',') }}{% if not loop.last %} {% endif %}{% endfor %}"/>
6
+ <text class="celltext" x="{{ line.text_x }}" y="{{ line.text_y }}">{{ line.text }}</text>
7
+ </a>
8
+ {% endfor %}
9
+ </svg>
app/gradio_config.py CHANGED
@@ -34,17 +34,29 @@ css = """
34
  overflow: visible;
35
  }
36
 
37
- /* style of textline svg elements */
38
- .textline {
39
  fill: transparent;
40
  stroke: blue;
41
  stroke-width: 10;
42
  stroke-opacity: 0.2;
43
  }
44
 
 
 
 
 
 
 
 
 
 
 
 
45
  .highlighted polygon {
46
  fill:blue;
47
- fill-opacity: 0.2;
 
48
  }
49
 
50
  span.highlighted {
 
34
  overflow: visible;
35
  }
36
 
37
+ /* style of table cell svg elements */
38
+ .cellline {
39
  fill: transparent;
40
  stroke: blue;
41
  stroke-width: 10;
42
  stroke-opacity: 0.2;
43
  }
44
 
45
+ svg > a > text {
46
+ fill: transparent;
47
+ stroke: transparent;
48
+ }
49
+
50
+ svg > a.highlighted > text {
51
+ fill: white;
52
+ stroke: transparent;
53
+ font-size: large;
54
+ }
55
+
56
  .highlighted polygon {
57
  fill:blue;
58
+ fill-opacity: 0.7;
59
+ stroke: black;
60
  }
61
 
62
  span.highlighted {
app/main.py CHANGED
@@ -2,6 +2,7 @@ import logging
2
  import os
3
 
4
  import gradio as gr
 
5
  # from htrflow.models.huggingface.trocr import TrOCR
6
 
7
  from app.gradio_config import css, theme
@@ -10,8 +11,8 @@ from app.gradio_config import css, theme
10
  # from app.tabs.export import export
11
  from app.tabs.submit import collection_submit_state, submit
12
 
13
- # from app.tabs.visualizer import collection as collection_viz_state
14
- # from app.tabs.visualizer import visualizer
15
 
16
  # Suppress transformers logging
17
  logging.getLogger("transformers").setLevel(logging.ERROR)
@@ -54,8 +55,8 @@ with gr.Blocks(title="Dawsonia Demo", theme=theme, css=css, head=html_header) as
54
  with gr.Tab(label="Upload") as tab_submit:
55
  submit.render()
56
 
57
- # with gr.Tab(label="Result", interactive=False, id="result") as tab_visualizer:
58
- # visualizer.render()
59
  #
60
  # with gr.Tab(label="Export", interactive=False) as tab_export:
61
  # export.render()
@@ -69,17 +70,17 @@ with gr.Blocks(title="Dawsonia Demo", theme=theme, css=css, head=html_header) as
69
  state_value = input_value
70
  return state_value if state_value is not None else gr.skip()
71
 
72
- # collection_submit_state.change(
73
- # activate_tab, collection_submit_state, tab_visualizer
74
- # )
75
  # collection_submit_state.change(activate_tab, collection_submit_state, tab_export)
76
  collection_submit_state.change(lambda: gr.Tabs(selected="result"), outputs=navbar)
77
 
78
- # tab_visualizer.select(
79
- # inputs=[collection_submit_state, collection_viz_state],
80
- # outputs=[collection_viz_state],
81
- # fn=sync_gradio_object_state,
82
- # )
83
  #
84
  # tab_export.select(
85
  # inputs=[collection_submit_state, collection_export_state],
@@ -91,5 +92,9 @@ demo.queue()
91
 
92
  if __name__ == "__main__":
93
  demo.launch(
94
- server_name="0.0.0.0", server_port=7860, enable_monitoring=True, show_api=False
 
 
 
 
95
  )
 
2
  import os
3
 
4
  import gradio as gr
5
+
6
  # from htrflow.models.huggingface.trocr import TrOCR
7
 
8
  from app.gradio_config import css, theme
 
11
  # from app.tabs.export import export
12
  from app.tabs.submit import collection_submit_state, submit
13
 
14
+ from app.tabs.visualizer import collection as collection_viz_state
15
+ from app.tabs.visualizer import visualizer
16
 
17
  # Suppress transformers logging
18
  logging.getLogger("transformers").setLevel(logging.ERROR)
 
55
  with gr.Tab(label="Upload") as tab_submit:
56
  submit.render()
57
 
58
+ with gr.Tab(label="Result", interactive=False, id="result") as tab_visualizer:
59
+ visualizer.render()
60
  #
61
  # with gr.Tab(label="Export", interactive=False) as tab_export:
62
  # export.render()
 
70
  state_value = input_value
71
  return state_value if state_value is not None else gr.skip()
72
 
73
+ collection_submit_state.change(
74
+ activate_tab, collection_submit_state, tab_visualizer
75
+ )
76
  # collection_submit_state.change(activate_tab, collection_submit_state, tab_export)
77
  collection_submit_state.change(lambda: gr.Tabs(selected="result"), outputs=navbar)
78
 
79
+ tab_visualizer.select(
80
+ inputs=[collection_submit_state, collection_viz_state],
81
+ outputs=[collection_viz_state],
82
+ fn=sync_gradio_object_state,
83
+ )
84
  #
85
  # tab_export.select(
86
  # inputs=[collection_submit_state, collection_export_state],
 
92
 
93
  if __name__ == "__main__":
94
  demo.launch(
95
+ server_name="0.0.0.0",
96
+ server_port=7860,
97
+ enable_monitoring=True,
98
+ show_api=False,
99
+ allowed_paths=[".gradio_cache/output"],
100
  )
app/tabs/submit.py CHANGED
@@ -3,6 +3,7 @@ import logging
3
  import os
4
  from pathlib import Path
5
  import time
 
6
 
7
  from PIL import Image
8
  from dawsonia import io
@@ -26,7 +27,7 @@ MAX_IMAGES = int(os.environ.get("MAX_IMAGES", 5))
26
  # Setup the cache directory to point to the directory where the example images
27
  # are located. The images must lay in the cache directory because otherwise they
28
  # have to be reuploaded when drag-and-dropped to the input image widget.
29
- GRADIO_CACHE = ".gradio_cache"
30
  DATA_CACHE = os.path.join(GRADIO_CACHE, "data")
31
  EXAMPLES_DIRECTORY = os.path.join(os.getcwd(), "examples")
32
 
@@ -38,17 +39,13 @@ PIPELINES: dict[str, dict[str, str]] = {
38
  )
39
  }
40
 
41
- if os.environ.get("GRADIO_CACHE_DIR", GRADIO_CACHE) != GRADIO_CACHE:
42
- os.environ["GRADIO_CACHE_DIR"] = GRADIO_CACHE
43
- logger.warning("Setting GRADIO_CACHE_DIR to '%s' (overriding a previous value).")
44
-
45
 
46
  def run_dawsonia(
47
- table_fmt_config_override, first_page, last_page, book, progress=gr.Progress()
48
  ):
49
  if book is None:
50
  raise ValueError("You need to select / upload the pages to digitize")
51
-
52
  progress(0, desc="Dawsonia: starting")
53
 
54
  model_path = Path("data/models/dawsonia/2024-07-02")
@@ -70,59 +67,70 @@ def run_dawsonia(
70
  ]
71
 
72
  collection = []
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
- for page_number in range(first_page, last_page):
75
- output_path_page = output_path_book / str(page_number)
76
- gr.Info(f"Digitizing {page_number = }")
77
-
78
- *_, stats = digitize.digitize_page_and_write_output(
79
- book,
80
- init_data,
81
- page_number=page_number,
82
- date_str=f"0000-page-{page_number}",
83
- model_path=model_path,
84
- model_predict=ml.model_predict,
85
- prob_thresh=0.5,
86
- output_path_page=output_path_page,
87
- output_text_fmt=False,
88
- debug=False,
89
- )
90
- progress_value = (page_number - first_page) / max(1, last_page - first_page)
91
- progress(progress_value, desc=f"Dawsonia: {stats!s:.50}")
92
 
93
- collection.append(read_page(stats, output_path_book, str(page_number)))
94
 
95
  gr.Info("Pages were succesfully digitized ✨")
96
 
 
97
  yield collection, gr.skip()
98
 
99
 
100
- def read_page(stats: digitize.Statistics, output_path_book: Path, prefix: str):
 
 
 
101
  if stats.tables_detected > 0:
102
  values_df = pd.read_parquet((output_path_book / prefix).with_suffix(".parquet"))
103
  table_meta = json.loads(
104
  (output_path_book / "table_meta" / prefix).with_suffix(".json").read_text()
105
  )
106
  with Image.open(
107
- (output_path_book / "pages" / prefix).with_suffix(".webp")
108
  ) as im:
109
  width = im.width
110
  height = im.height
111
 
112
  values_array = values_df.values.flatten()
113
- bbox_array = np.array(table_meta["table_positions"]).reshape(
114
- values_array.size, 4
115
  )
116
  cells = [
117
  make_cell(value, bbox) for value, bbox in zip(values_array, bbox_array)
118
  ]
119
- return Page(width, height, cells)
120
 
121
 
122
  def make_cell(value: str, bbox: NDArray[np.int64]):
123
- x, y, w, h = bbox
124
- xmax, ymax = x+w, y+h
125
- polygon = (x,y), (xmax, y), (xmax, ymax), (x, ymax), (x,y)
 
126
  return TableCell(polygon, text_x=x, text_y=y, text=value)
127
 
128
 
@@ -229,11 +237,12 @@ with gr.Blocks() as submit:
229
  if len(images) > MAX_IMAGES:
230
  gr.Warning(f"Maximum images you can upload is set to: {MAX_IMAGES}")
231
  return gr.update(value=None)
 
232
  return images
233
-
234
  run_button.click(
235
  fn=run_dawsonia,
236
- inputs=(table_fmt_config_override, first_page, last_page, batch_book_state),
237
  outputs=(collection_submit_state, batch_image_gallery),
238
  )
239
  edit_table_fmt_button.click(lambda: Modal(visible=True), None, edit_table_fmt_modal)
 
3
  import os
4
  from pathlib import Path
5
  import time
6
+ import warnings
7
 
8
  from PIL import Image
9
  from dawsonia import io
 
27
  # Setup the cache directory to point to the directory where the example images
28
  # are located. The images must lay in the cache directory because otherwise they
29
  # have to be reuploaded when drag-and-dropped to the input image widget.
30
+ GRADIO_CACHE = os.getenv("GRADIO_CACHE_DIR", ".gradio_cache")
31
  DATA_CACHE = os.path.join(GRADIO_CACHE, "data")
32
  EXAMPLES_DIRECTORY = os.path.join(os.getcwd(), "examples")
33
 
 
39
  )
40
  }
41
 
 
 
 
 
42
 
43
  def run_dawsonia(
44
+ table_fmt_config_override, first_page, last_page, book, gallery, progress=gr.Progress()
45
  ):
46
  if book is None:
47
  raise ValueError("You need to select / upload the pages to digitize")
48
+
49
  progress(0, desc="Dawsonia: starting")
50
 
51
  model_path = Path("data/models/dawsonia/2024-07-02")
 
67
  ]
68
 
69
  collection = []
70
+ images = []
71
+
72
+ with warnings.catch_warnings():
73
+ warnings.simplefilter("ignore", FutureWarning)
74
+ for page_number, im_from_gallery in zip(range(first_page, last_page), gallery):
75
+ output_path_page = output_path_book / str(page_number)
76
+ gr.Info(f"Digitizing {page_number = }")
77
+
78
+ if not (output_path_book / str(page_number)).with_suffix(".parquet").exists():
79
+ digitize.digitize_page_and_write_output(
80
+ book,
81
+ init_data,
82
+ page_number=page_number,
83
+ date_str=f"0000-page-{page_number}",
84
+ model_path=model_path,
85
+ model_predict=ml.model_predict,
86
+ prob_thresh=0.5,
87
+ output_path_page=output_path_page,
88
+ output_text_fmt=False,
89
+ debug=False,
90
+ )
91
+ progress_value = (page_number - first_page) / max(1, last_page - first_page)
92
 
93
+ page, im = read_page(output_path_book, str(page_number), progress, progress_value) # , im_from_gallery[0])
94
+ collection.append(page)
95
+ images.append(im)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
96
 
 
97
 
98
  gr.Info("Pages were succesfully digitized ✨")
99
 
100
+ # yield collection, images
101
  yield collection, gr.skip()
102
 
103
 
104
+ def read_page(output_path_book: Path, prefix: str, progress, progress_value, im_path_from_gallery: str = ""):
105
+ stats = digitize.Statistics.from_json((output_path_book / "statistics" / prefix).with_suffix(".json"))
106
+ print(stats)
107
+ progress(progress_value, desc=f"Dawsonia: {stats!s:.50}")
108
  if stats.tables_detected > 0:
109
  values_df = pd.read_parquet((output_path_book / prefix).with_suffix(".parquet"))
110
  table_meta = json.loads(
111
  (output_path_book / "table_meta" / prefix).with_suffix(".json").read_text()
112
  )
113
  with Image.open(
114
+ image_path:=(output_path_book / "pages" / prefix).with_suffix(".webp")
115
  ) as im:
116
  width = im.width
117
  height = im.height
118
 
119
  values_array = values_df.values.flatten()
120
+ bbox_array = np.hstack(table_meta["table_positions"]).reshape(
121
+ -1, 4
122
  )
123
  cells = [
124
  make_cell(value, bbox) for value, bbox in zip(values_array, bbox_array)
125
  ]
126
+ return Page(width, height, cells, im_path_from_gallery or str(image_path)), im
127
 
128
 
129
  def make_cell(value: str, bbox: NDArray[np.int64]):
130
+ y, x, h, w = bbox
131
+ xmin, ymin = x-w//2, y-h//2
132
+ xmax, ymax = x+w//2, y+h//2
133
+ polygon = (xmin,ymin), (xmax, ymin), (xmax, ymax), (xmin, ymax), (xmin,ymin)
134
  return TableCell(polygon, text_x=x, text_y=y, text=value)
135
 
136
 
 
237
  if len(images) > MAX_IMAGES:
238
  gr.Warning(f"Maximum images you can upload is set to: {MAX_IMAGES}")
239
  return gr.update(value=None)
240
+
241
  return images
242
+
243
  run_button.click(
244
  fn=run_dawsonia,
245
+ inputs=(table_fmt_config_override, first_page, last_page, batch_book_state, batch_image_gallery),
246
  outputs=(collection_submit_state, batch_image_gallery),
247
  )
248
  edit_table_fmt_button.click(lambda: Modal(visible=True), None, edit_table_fmt_modal)
app/tabs/visualizer.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ from jinja2 import Environment, FileSystemLoader
3
+
4
+ _ENV = Environment(loader=FileSystemLoader("app/assets/jinja-templates"))
5
+ _IMAGE_TEMPLATE = _ENV.get_template("image.j2")
6
+
7
+ from typing import NamedTuple
8
+ from dawsonia.typing import BBoxTuple
9
+
10
+
11
+ class TableCell(NamedTuple):
12
+ polygon: tuple[tuple[int, int], ...]
13
+ text_x: int
14
+ text_y: int
15
+ text: str
16
+
17
+
18
+ class Page(NamedTuple):
19
+ width: int
20
+ height: int
21
+ cells: list[TableCell]
22
+ path: str
23
+
24
+
25
+ def render_image(collection: list[Page], current_page_index: int) -> str:
26
+ return _IMAGE_TEMPLATE.render(
27
+ page=collection[current_page_index],
28
+ )
29
+
30
+
31
+ with gr.Blocks() as visualizer:
32
+ gr.Markdown("# Result")
33
+ gr.Markdown(
34
+ "The image to the below shows where Dawsonia found text in the image."
35
+ )
36
+
37
+ with gr.Row():
38
+ # Annotated image panel
39
+ with gr.Column(scale=2):
40
+ image = gr.HTML(
41
+ label="Annotated image",
42
+ padding=False,
43
+ elem_classes="svg-image",
44
+ container=True,
45
+ max_height="65vh",
46
+ min_height="65vh",
47
+ show_label=True,
48
+ )
49
+
50
+ image_caption = gr.Markdown(elem_classes="button-group-viz")
51
+ with gr.Row(elem_classes="button-group-viz"):
52
+ left = gr.Button(
53
+ "← Previous", visible=False, interactive=False, scale=0
54
+ )
55
+ right = gr.Button("Next →", visible=False, scale=0)
56
+
57
+ collection = gr.State()
58
+ current_page_index = gr.State(0)
59
+
60
+ # Updates on collection change:
61
+ # - update the view
62
+ # - reset the page index (always start on page 0)
63
+ # - toggle visibility of navigation buttons (don't show them for single pages)
64
+ # - update the image caption
65
+ collection.change(
66
+ render_image, inputs=[collection, current_page_index], outputs=image
67
+ )
examples/bjur/303/266klubb.png ADDED