Spaces:
Running
Running
Ashwin V. Mohanan
commited on
Commit
·
97ba84c
1
Parent(s):
9714711
Digitize and visualize
Browse files- app/assets/jinja-templates/image.j2 +9 -0
- app/gradio_config.py +15 -3
- app/main.py +18 -13
- app/tabs/submit.py +45 -36
- app/tabs/visualizer.py +67 -0
- examples/bjur/303/266klubb.png +0 -0
app/assets/jinja-templates/image.j2
ADDED
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
<svg viewBox="0 0 {{ page.width }} {{ page.height }}" xmlns="http://www.w3.org/2000/svg">
|
2 |
+
<image height="{{ page.height }}" width="{{ page.width }}" href="/gradio_api/file={{ page.path }}" />
|
3 |
+
{%- for line in page.cells -%}
|
4 |
+
<a class="cellline line{{loop.index}} highlighted" onmouseover="document.querySelectorAll('.line{{loop.index}}').forEach(element => {element.classList.remove('highlighted')});" onmouseout="document.querySelectorAll('.line{{loop.index}}').forEach(element => {element.classList.add('highlighted')});">
|
5 |
+
<polygon id="{{ loop.index }}" points="{% for point in line.polygon %}{{ point|join(',') }}{% if not loop.last %} {% endif %}{% endfor %}"/>
|
6 |
+
<text class="celltext" x="{{ line.text_x }}" y="{{ line.text_y }}">{{ line.text }}</text>
|
7 |
+
</a>
|
8 |
+
{% endfor %}
|
9 |
+
</svg>
|
app/gradio_config.py
CHANGED
@@ -34,17 +34,29 @@ css = """
|
|
34 |
overflow: visible;
|
35 |
}
|
36 |
|
37 |
-
/* style of
|
38 |
-
.
|
39 |
fill: transparent;
|
40 |
stroke: blue;
|
41 |
stroke-width: 10;
|
42 |
stroke-opacity: 0.2;
|
43 |
}
|
44 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
45 |
.highlighted polygon {
|
46 |
fill:blue;
|
47 |
-
fill-opacity: 0.
|
|
|
48 |
}
|
49 |
|
50 |
span.highlighted {
|
|
|
34 |
overflow: visible;
|
35 |
}
|
36 |
|
37 |
+
/* style of table cell svg elements */
|
38 |
+
.cellline {
|
39 |
fill: transparent;
|
40 |
stroke: blue;
|
41 |
stroke-width: 10;
|
42 |
stroke-opacity: 0.2;
|
43 |
}
|
44 |
|
45 |
+
svg > a > text {
|
46 |
+
fill: transparent;
|
47 |
+
stroke: transparent;
|
48 |
+
}
|
49 |
+
|
50 |
+
svg > a.highlighted > text {
|
51 |
+
fill: white;
|
52 |
+
stroke: transparent;
|
53 |
+
font-size: large;
|
54 |
+
}
|
55 |
+
|
56 |
.highlighted polygon {
|
57 |
fill:blue;
|
58 |
+
fill-opacity: 0.7;
|
59 |
+
stroke: black;
|
60 |
}
|
61 |
|
62 |
span.highlighted {
|
app/main.py
CHANGED
@@ -2,6 +2,7 @@ import logging
|
|
2 |
import os
|
3 |
|
4 |
import gradio as gr
|
|
|
5 |
# from htrflow.models.huggingface.trocr import TrOCR
|
6 |
|
7 |
from app.gradio_config import css, theme
|
@@ -10,8 +11,8 @@ from app.gradio_config import css, theme
|
|
10 |
# from app.tabs.export import export
|
11 |
from app.tabs.submit import collection_submit_state, submit
|
12 |
|
13 |
-
|
14 |
-
|
15 |
|
16 |
# Suppress transformers logging
|
17 |
logging.getLogger("transformers").setLevel(logging.ERROR)
|
@@ -54,8 +55,8 @@ with gr.Blocks(title="Dawsonia Demo", theme=theme, css=css, head=html_header) as
|
|
54 |
with gr.Tab(label="Upload") as tab_submit:
|
55 |
submit.render()
|
56 |
|
57 |
-
|
58 |
-
|
59 |
#
|
60 |
# with gr.Tab(label="Export", interactive=False) as tab_export:
|
61 |
# export.render()
|
@@ -69,17 +70,17 @@ with gr.Blocks(title="Dawsonia Demo", theme=theme, css=css, head=html_header) as
|
|
69 |
state_value = input_value
|
70 |
return state_value if state_value is not None else gr.skip()
|
71 |
|
72 |
-
|
73 |
-
|
74 |
-
|
75 |
# collection_submit_state.change(activate_tab, collection_submit_state, tab_export)
|
76 |
collection_submit_state.change(lambda: gr.Tabs(selected="result"), outputs=navbar)
|
77 |
|
78 |
-
|
79 |
-
|
80 |
-
|
81 |
-
|
82 |
-
|
83 |
#
|
84 |
# tab_export.select(
|
85 |
# inputs=[collection_submit_state, collection_export_state],
|
@@ -91,5 +92,9 @@ demo.queue()
|
|
91 |
|
92 |
if __name__ == "__main__":
|
93 |
demo.launch(
|
94 |
-
server_name="0.0.0.0",
|
|
|
|
|
|
|
|
|
95 |
)
|
|
|
2 |
import os
|
3 |
|
4 |
import gradio as gr
|
5 |
+
|
6 |
# from htrflow.models.huggingface.trocr import TrOCR
|
7 |
|
8 |
from app.gradio_config import css, theme
|
|
|
11 |
# from app.tabs.export import export
|
12 |
from app.tabs.submit import collection_submit_state, submit
|
13 |
|
14 |
+
from app.tabs.visualizer import collection as collection_viz_state
|
15 |
+
from app.tabs.visualizer import visualizer
|
16 |
|
17 |
# Suppress transformers logging
|
18 |
logging.getLogger("transformers").setLevel(logging.ERROR)
|
|
|
55 |
with gr.Tab(label="Upload") as tab_submit:
|
56 |
submit.render()
|
57 |
|
58 |
+
with gr.Tab(label="Result", interactive=False, id="result") as tab_visualizer:
|
59 |
+
visualizer.render()
|
60 |
#
|
61 |
# with gr.Tab(label="Export", interactive=False) as tab_export:
|
62 |
# export.render()
|
|
|
70 |
state_value = input_value
|
71 |
return state_value if state_value is not None else gr.skip()
|
72 |
|
73 |
+
collection_submit_state.change(
|
74 |
+
activate_tab, collection_submit_state, tab_visualizer
|
75 |
+
)
|
76 |
# collection_submit_state.change(activate_tab, collection_submit_state, tab_export)
|
77 |
collection_submit_state.change(lambda: gr.Tabs(selected="result"), outputs=navbar)
|
78 |
|
79 |
+
tab_visualizer.select(
|
80 |
+
inputs=[collection_submit_state, collection_viz_state],
|
81 |
+
outputs=[collection_viz_state],
|
82 |
+
fn=sync_gradio_object_state,
|
83 |
+
)
|
84 |
#
|
85 |
# tab_export.select(
|
86 |
# inputs=[collection_submit_state, collection_export_state],
|
|
|
92 |
|
93 |
if __name__ == "__main__":
|
94 |
demo.launch(
|
95 |
+
server_name="0.0.0.0",
|
96 |
+
server_port=7860,
|
97 |
+
enable_monitoring=True,
|
98 |
+
show_api=False,
|
99 |
+
allowed_paths=[".gradio_cache/output"],
|
100 |
)
|
app/tabs/submit.py
CHANGED
@@ -3,6 +3,7 @@ import logging
|
|
3 |
import os
|
4 |
from pathlib import Path
|
5 |
import time
|
|
|
6 |
|
7 |
from PIL import Image
|
8 |
from dawsonia import io
|
@@ -26,7 +27,7 @@ MAX_IMAGES = int(os.environ.get("MAX_IMAGES", 5))
|
|
26 |
# Setup the cache directory to point to the directory where the example images
|
27 |
# are located. The images must lay in the cache directory because otherwise they
|
28 |
# have to be reuploaded when drag-and-dropped to the input image widget.
|
29 |
-
GRADIO_CACHE = ".gradio_cache"
|
30 |
DATA_CACHE = os.path.join(GRADIO_CACHE, "data")
|
31 |
EXAMPLES_DIRECTORY = os.path.join(os.getcwd(), "examples")
|
32 |
|
@@ -38,17 +39,13 @@ PIPELINES: dict[str, dict[str, str]] = {
|
|
38 |
)
|
39 |
}
|
40 |
|
41 |
-
if os.environ.get("GRADIO_CACHE_DIR", GRADIO_CACHE) != GRADIO_CACHE:
|
42 |
-
os.environ["GRADIO_CACHE_DIR"] = GRADIO_CACHE
|
43 |
-
logger.warning("Setting GRADIO_CACHE_DIR to '%s' (overriding a previous value).")
|
44 |
-
|
45 |
|
46 |
def run_dawsonia(
|
47 |
-
table_fmt_config_override, first_page, last_page, book, progress=gr.Progress()
|
48 |
):
|
49 |
if book is None:
|
50 |
raise ValueError("You need to select / upload the pages to digitize")
|
51 |
-
|
52 |
progress(0, desc="Dawsonia: starting")
|
53 |
|
54 |
model_path = Path("data/models/dawsonia/2024-07-02")
|
@@ -70,59 +67,70 @@ def run_dawsonia(
|
|
70 |
]
|
71 |
|
72 |
collection = []
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
73 |
|
74 |
-
|
75 |
-
|
76 |
-
|
77 |
-
|
78 |
-
*_, stats = digitize.digitize_page_and_write_output(
|
79 |
-
book,
|
80 |
-
init_data,
|
81 |
-
page_number=page_number,
|
82 |
-
date_str=f"0000-page-{page_number}",
|
83 |
-
model_path=model_path,
|
84 |
-
model_predict=ml.model_predict,
|
85 |
-
prob_thresh=0.5,
|
86 |
-
output_path_page=output_path_page,
|
87 |
-
output_text_fmt=False,
|
88 |
-
debug=False,
|
89 |
-
)
|
90 |
-
progress_value = (page_number - first_page) / max(1, last_page - first_page)
|
91 |
-
progress(progress_value, desc=f"Dawsonia: {stats!s:.50}")
|
92 |
|
93 |
-
collection.append(read_page(stats, output_path_book, str(page_number)))
|
94 |
|
95 |
gr.Info("Pages were succesfully digitized ✨")
|
96 |
|
|
|
97 |
yield collection, gr.skip()
|
98 |
|
99 |
|
100 |
-
def read_page(
|
|
|
|
|
|
|
101 |
if stats.tables_detected > 0:
|
102 |
values_df = pd.read_parquet((output_path_book / prefix).with_suffix(".parquet"))
|
103 |
table_meta = json.loads(
|
104 |
(output_path_book / "table_meta" / prefix).with_suffix(".json").read_text()
|
105 |
)
|
106 |
with Image.open(
|
107 |
-
(output_path_book / "pages" / prefix).with_suffix(".webp")
|
108 |
) as im:
|
109 |
width = im.width
|
110 |
height = im.height
|
111 |
|
112 |
values_array = values_df.values.flatten()
|
113 |
-
bbox_array = np.
|
114 |
-
|
115 |
)
|
116 |
cells = [
|
117 |
make_cell(value, bbox) for value, bbox in zip(values_array, bbox_array)
|
118 |
]
|
119 |
-
return Page(width, height, cells)
|
120 |
|
121 |
|
122 |
def make_cell(value: str, bbox: NDArray[np.int64]):
|
123 |
-
x,
|
124 |
-
|
125 |
-
|
|
|
126 |
return TableCell(polygon, text_x=x, text_y=y, text=value)
|
127 |
|
128 |
|
@@ -229,11 +237,12 @@ with gr.Blocks() as submit:
|
|
229 |
if len(images) > MAX_IMAGES:
|
230 |
gr.Warning(f"Maximum images you can upload is set to: {MAX_IMAGES}")
|
231 |
return gr.update(value=None)
|
|
|
232 |
return images
|
233 |
-
|
234 |
run_button.click(
|
235 |
fn=run_dawsonia,
|
236 |
-
inputs=(table_fmt_config_override, first_page, last_page, batch_book_state),
|
237 |
outputs=(collection_submit_state, batch_image_gallery),
|
238 |
)
|
239 |
edit_table_fmt_button.click(lambda: Modal(visible=True), None, edit_table_fmt_modal)
|
|
|
3 |
import os
|
4 |
from pathlib import Path
|
5 |
import time
|
6 |
+
import warnings
|
7 |
|
8 |
from PIL import Image
|
9 |
from dawsonia import io
|
|
|
27 |
# Setup the cache directory to point to the directory where the example images
|
28 |
# are located. The images must lay in the cache directory because otherwise they
|
29 |
# have to be reuploaded when drag-and-dropped to the input image widget.
|
30 |
+
GRADIO_CACHE = os.getenv("GRADIO_CACHE_DIR", ".gradio_cache")
|
31 |
DATA_CACHE = os.path.join(GRADIO_CACHE, "data")
|
32 |
EXAMPLES_DIRECTORY = os.path.join(os.getcwd(), "examples")
|
33 |
|
|
|
39 |
)
|
40 |
}
|
41 |
|
|
|
|
|
|
|
|
|
42 |
|
43 |
def run_dawsonia(
|
44 |
+
table_fmt_config_override, first_page, last_page, book, gallery, progress=gr.Progress()
|
45 |
):
|
46 |
if book is None:
|
47 |
raise ValueError("You need to select / upload the pages to digitize")
|
48 |
+
|
49 |
progress(0, desc="Dawsonia: starting")
|
50 |
|
51 |
model_path = Path("data/models/dawsonia/2024-07-02")
|
|
|
67 |
]
|
68 |
|
69 |
collection = []
|
70 |
+
images = []
|
71 |
+
|
72 |
+
with warnings.catch_warnings():
|
73 |
+
warnings.simplefilter("ignore", FutureWarning)
|
74 |
+
for page_number, im_from_gallery in zip(range(first_page, last_page), gallery):
|
75 |
+
output_path_page = output_path_book / str(page_number)
|
76 |
+
gr.Info(f"Digitizing {page_number = }")
|
77 |
+
|
78 |
+
if not (output_path_book / str(page_number)).with_suffix(".parquet").exists():
|
79 |
+
digitize.digitize_page_and_write_output(
|
80 |
+
book,
|
81 |
+
init_data,
|
82 |
+
page_number=page_number,
|
83 |
+
date_str=f"0000-page-{page_number}",
|
84 |
+
model_path=model_path,
|
85 |
+
model_predict=ml.model_predict,
|
86 |
+
prob_thresh=0.5,
|
87 |
+
output_path_page=output_path_page,
|
88 |
+
output_text_fmt=False,
|
89 |
+
debug=False,
|
90 |
+
)
|
91 |
+
progress_value = (page_number - first_page) / max(1, last_page - first_page)
|
92 |
|
93 |
+
page, im = read_page(output_path_book, str(page_number), progress, progress_value) # , im_from_gallery[0])
|
94 |
+
collection.append(page)
|
95 |
+
images.append(im)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
96 |
|
|
|
97 |
|
98 |
gr.Info("Pages were succesfully digitized ✨")
|
99 |
|
100 |
+
# yield collection, images
|
101 |
yield collection, gr.skip()
|
102 |
|
103 |
|
104 |
+
def read_page(output_path_book: Path, prefix: str, progress, progress_value, im_path_from_gallery: str = ""):
|
105 |
+
stats = digitize.Statistics.from_json((output_path_book / "statistics" / prefix).with_suffix(".json"))
|
106 |
+
print(stats)
|
107 |
+
progress(progress_value, desc=f"Dawsonia: {stats!s:.50}")
|
108 |
if stats.tables_detected > 0:
|
109 |
values_df = pd.read_parquet((output_path_book / prefix).with_suffix(".parquet"))
|
110 |
table_meta = json.loads(
|
111 |
(output_path_book / "table_meta" / prefix).with_suffix(".json").read_text()
|
112 |
)
|
113 |
with Image.open(
|
114 |
+
image_path:=(output_path_book / "pages" / prefix).with_suffix(".webp")
|
115 |
) as im:
|
116 |
width = im.width
|
117 |
height = im.height
|
118 |
|
119 |
values_array = values_df.values.flatten()
|
120 |
+
bbox_array = np.hstack(table_meta["table_positions"]).reshape(
|
121 |
+
-1, 4
|
122 |
)
|
123 |
cells = [
|
124 |
make_cell(value, bbox) for value, bbox in zip(values_array, bbox_array)
|
125 |
]
|
126 |
+
return Page(width, height, cells, im_path_from_gallery or str(image_path)), im
|
127 |
|
128 |
|
129 |
def make_cell(value: str, bbox: NDArray[np.int64]):
|
130 |
+
y, x, h, w = bbox
|
131 |
+
xmin, ymin = x-w//2, y-h//2
|
132 |
+
xmax, ymax = x+w//2, y+h//2
|
133 |
+
polygon = (xmin,ymin), (xmax, ymin), (xmax, ymax), (xmin, ymax), (xmin,ymin)
|
134 |
return TableCell(polygon, text_x=x, text_y=y, text=value)
|
135 |
|
136 |
|
|
|
237 |
if len(images) > MAX_IMAGES:
|
238 |
gr.Warning(f"Maximum images you can upload is set to: {MAX_IMAGES}")
|
239 |
return gr.update(value=None)
|
240 |
+
|
241 |
return images
|
242 |
+
|
243 |
run_button.click(
|
244 |
fn=run_dawsonia,
|
245 |
+
inputs=(table_fmt_config_override, first_page, last_page, batch_book_state, batch_image_gallery),
|
246 |
outputs=(collection_submit_state, batch_image_gallery),
|
247 |
)
|
248 |
edit_table_fmt_button.click(lambda: Modal(visible=True), None, edit_table_fmt_modal)
|
app/tabs/visualizer.py
ADDED
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import gradio as gr
|
2 |
+
from jinja2 import Environment, FileSystemLoader
|
3 |
+
|
4 |
+
_ENV = Environment(loader=FileSystemLoader("app/assets/jinja-templates"))
|
5 |
+
_IMAGE_TEMPLATE = _ENV.get_template("image.j2")
|
6 |
+
|
7 |
+
from typing import NamedTuple
|
8 |
+
from dawsonia.typing import BBoxTuple
|
9 |
+
|
10 |
+
|
11 |
+
class TableCell(NamedTuple):
|
12 |
+
polygon: tuple[tuple[int, int], ...]
|
13 |
+
text_x: int
|
14 |
+
text_y: int
|
15 |
+
text: str
|
16 |
+
|
17 |
+
|
18 |
+
class Page(NamedTuple):
|
19 |
+
width: int
|
20 |
+
height: int
|
21 |
+
cells: list[TableCell]
|
22 |
+
path: str
|
23 |
+
|
24 |
+
|
25 |
+
def render_image(collection: list[Page], current_page_index: int) -> str:
|
26 |
+
return _IMAGE_TEMPLATE.render(
|
27 |
+
page=collection[current_page_index],
|
28 |
+
)
|
29 |
+
|
30 |
+
|
31 |
+
with gr.Blocks() as visualizer:
|
32 |
+
gr.Markdown("# Result")
|
33 |
+
gr.Markdown(
|
34 |
+
"The image to the below shows where Dawsonia found text in the image."
|
35 |
+
)
|
36 |
+
|
37 |
+
with gr.Row():
|
38 |
+
# Annotated image panel
|
39 |
+
with gr.Column(scale=2):
|
40 |
+
image = gr.HTML(
|
41 |
+
label="Annotated image",
|
42 |
+
padding=False,
|
43 |
+
elem_classes="svg-image",
|
44 |
+
container=True,
|
45 |
+
max_height="65vh",
|
46 |
+
min_height="65vh",
|
47 |
+
show_label=True,
|
48 |
+
)
|
49 |
+
|
50 |
+
image_caption = gr.Markdown(elem_classes="button-group-viz")
|
51 |
+
with gr.Row(elem_classes="button-group-viz"):
|
52 |
+
left = gr.Button(
|
53 |
+
"← Previous", visible=False, interactive=False, scale=0
|
54 |
+
)
|
55 |
+
right = gr.Button("Next →", visible=False, scale=0)
|
56 |
+
|
57 |
+
collection = gr.State()
|
58 |
+
current_page_index = gr.State(0)
|
59 |
+
|
60 |
+
# Updates on collection change:
|
61 |
+
# - update the view
|
62 |
+
# - reset the page index (always start on page 0)
|
63 |
+
# - toggle visibility of navigation buttons (don't show them for single pages)
|
64 |
+
# - update the image caption
|
65 |
+
collection.change(
|
66 |
+
render_image, inputs=[collection, current_page_index], outputs=image
|
67 |
+
)
|
examples/bjur/303/266klubb.png
ADDED
![]() |