Add first rough version

app/content/main_sub_title.md

@@ -0,0 +1,3 @@
+<a href="https://dawsonia.readthedocs.io">
+<img src="https://git.smhi.se/ai-for-obs/dawsonia/-/raw/main/docs/source/_static/smhi_logo_black_8mm.png" width="17%" align="right" margin-right="200" />
+</a>

app/content/main_title.md

@@ -0,0 +1 @@
+<h1><center> Dawsonia 🔍 App </center></h1>

app/gradio_config.py

@@ -0,0 +1,118 @@
+import gradio as gr
+
+theme = gr.themes.Default(
+    primary_hue="blue",
+    secondary_hue="blue",
+    neutral_hue="slate",
+    # font=[
+    #     gr.themes.GoogleFont("Open Sans"),
+    #     "ui-sans-serif",
+    #     "system-ui",
+    #     "sans-serif",
+    # ],
+)
+
+css = """
+.svg-image {
+  height: auto;
+  width: 100%;
+  margin: auto;
+}
+
+.transcription {
+  font-size: large;
+  position: sticky;
+  top: 20px;
+}
+
+.transcription-column {
+  height: 100vh;
+}
+
+/* this is needed in order to make the transcription sticky */
+.app {
+  overflow: visible;
+}
+
+/* style of textline svg elements */
+.textline {
+  fill: transparent;
+  stroke: blue;
+  stroke-width: 10;
+  stroke-opacity: 0.2;
+}
+
+.highlighted polygon {
+  fill:blue;
+  fill-opacity: 0.2;
+}
+
+span.highlighted {
+  background-color: rgba(0%, 0%, 100%, 0.2);
+  font-size: large;
+}
+
+hr.region-divider {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+
+.pipeline-panel {
+  background: none;
+  border: solid 1px;
+  border-color: var(--block-border-color);
+}
+
+.pipeline-help {
+  padding: 5px 0 0 0;
+  font-weight: var(--block-info-text-weight);
+  font-size: var(--block-info-text-size);
+  color: var(--block-info-text-color);
+}
+
+.pipeline-info {
+  padding: 0 0 0 2px;
+  font-weight: var(--block-info-text-weight);
+  font-size: var(--block-info-text-size);
+  color: var(--block-info-text-color);
+}
+
+.pipeline-help a {
+  color: var(--secondary-400);
+}
+
+.pipeline-help a:hover {
+  color: var(--secondary-500);
+}
+
+.pipeline-header {
+  padding: 2px 0px 0px 2px;
+  color: var(--body-text-color);
+}
+
+.pipeline-description {
+  margin: auto;
+  color: var(--body-text-color);
+}
+
+.button-group-viz {
+ margin: auto;
+ display: flex;
+ justify-content: center;
+ gap: 1rem;
+ text-align: center;
+}
+
+.modal-block {
+  width: 60%;
+  padding: 1rem;
+}
+
+@media (max-width: 1024px) { /* mobile and standing iPads */
+  .modal-block {
+    width: 100%;
+  }
+}
+
+
+"""

app/main.py

@@ -0,0 +1,95 @@
+import logging
+import os
+
+import gradio as gr
+# from htrflow.models.huggingface.trocr import TrOCR
+
+from app.gradio_config import css, theme
+
+# from app.tabs.export import collection as collection_export_state
+# from app.tabs.export import export
+from app.tabs.submit import collection_submit_state, submit
+
+# from app.tabs.visualizer import collection as collection_viz_state
+# from app.tabs.visualizer import visualizer
+
+# Suppress transformers logging
+logging.getLogger("transformers").setLevel(logging.ERROR)
+
+
+TEMPLATE_YAML_FOLDER = "app/assets/templates"
+gr.set_static_paths(paths=[TEMPLATE_YAML_FOLDER])
+
+
+def load_markdown(language, section, content_dir="app/content"):
+    """Load markdown content from files."""
+    if language is None:
+        file_path = os.path.join(content_dir, f"{section}.md")
+    else:
+        file_path = os.path.join(content_dir, language, f"{section}.md")
+
+    if os.path.exists(file_path):
+        with open(file_path, "r", encoding="utf-8") as f:
+            return f.read()
+    return f"## Content missing for {file_path} in {language}"
+
+
+def activate_tab(collection):
+    return gr.update(interactive=collection is not None)
+
+
+html_header = ""
+
+
+with gr.Blocks(title="Dawsonia Demo", theme=theme, css=css, head=html_header) as demo:
+    with gr.Row():
+        with gr.Column(scale=1):
+            pass
+        with gr.Column(scale=2):
+            gr.Markdown(load_markdown(None, "main_title"))
+        with gr.Column(scale=1):
+            gr.Markdown(load_markdown(None, "main_sub_title"))
+
+    with gr.Tabs(elem_classes="top-navbar") as navbar:
+        with gr.Tab(label="Upload") as tab_submit:
+            submit.render()
+
+        # with gr.Tab(label="Result", interactive=False, id="result") as tab_visualizer:
+        #     visualizer.render()
+        #
+        # with gr.Tab(label="Export", interactive=False) as tab_export:
+        #     export.render()
+
+    # @demo.load()
+    # def inital_trocr_load():
+    #     TrOCR("Riksarkivet/trocr-base-handwritten-hist-swe-2")
+
+    def sync_gradio_object_state(input_value, state_value):
+        """Synchronize the Collection."""
+        state_value = input_value
+        return state_value if state_value is not None else gr.skip()
+
+    # collection_submit_state.change(
+    #     activate_tab, collection_submit_state, tab_visualizer
+    # )
+    # collection_submit_state.change(activate_tab, collection_submit_state, tab_export)
+    collection_submit_state.change(lambda: gr.Tabs(selected="result"), outputs=navbar)
+
+    # tab_visualizer.select(
+    #     inputs=[collection_submit_state, collection_viz_state],
+    #     outputs=[collection_viz_state],
+    #     fn=sync_gradio_object_state,
+    # )
+    #
+    # tab_export.select(
+    #     inputs=[collection_submit_state, collection_export_state],
+    #     outputs=[collection_export_state],
+    #     fn=sync_gradio_object_state,
+    # )
+
+demo.queue()
+
+if __name__ == "__main__":
+    demo.launch(
+        server_name="0.0.0.0", server_port=7860, enable_monitoring=True, show_api=False
+    )

app/tabs/submit.py

@@ -0,0 +1,245 @@
+import logging
+import os
+import time
+
+import numpy as np
+import pandas as pd
+from pathlib import Path
+from dawsonia import io
+from dawsonia import digitize
+from dawsonia.ml import ml
+import pooch
+import gradio as gr
+import yaml
+from gradio_modal import Modal
+
+logger = logging.getLogger(__name__)
+
+# Max number of images a user can upload at once
+MAX_IMAGES = int(os.environ.get("MAX_IMAGES", 5))
+
+# Setup the cache directory to point to the directory where the example images
+# are located. The images must lay in the cache directory because otherwise they
+# have to be reuploaded when drag-and-dropped to the input image widget.
+GRADIO_CACHE = ".gradio_cache"
+DATA_CACHE = os.path.join(GRADIO_CACHE, "data")
+EXAMPLES_DIRECTORY = os.path.join(os.getcwd(), "examples")
+
+# Example books
+PIPELINES: dict[str, dict[str, str]] = {
+    "bjuröklubb": dict(
+        url="https://git.smhi.se/ai-for-obs/data/-/raw/688c04f13e8e946962792fe4b4e0ded98800b154/raw_zarr/BJUR%C3%96KLUBB/DAGBOK_Bjur%C3%B6klubb_Station_Jan-Dec_1928.zarr.zip",
+        known_hash="sha256:6d87b7f79836ae6373cfab11260fe28787d93fe16199fefede6697ccd750f71a",
+    )
+}
+
+if os.environ.get("GRADIO_CACHE_DIR", GRADIO_CACHE) != GRADIO_CACHE:
+    os.environ["GRADIO_CACHE_DIR"] = GRADIO_CACHE
+    logger.warning("Setting GRADIO_CACHE_DIR to '%s' (overriding a previous value).")
+
+
+def run_dawsonia(
+    table_fmt_config_override, batch_image_gallery, book, progress=gr.Progress()
+):
+    if None in (batch_image_gallery, book) or len(batch_image_gallery) == 0:
+        raise ValueError("You need to select / upload the pages to digitize")
+    progress(0, desc="Dawsonia: starting")
+
+    model_path = Path("data/models/dawsonia/2024-07-02")
+    output_path = Path(GRADIO_CACHE, "output")
+
+    print("Dawsonia: digitizing", book)
+    table_fmt = book.table_format
+
+    output_path_book = output_path / book.station_name / book._name
+    output_path_book.mkdir(exist_ok=True, parents=True)
+    (output_path_book / "probablities").mkdir(exist_ok=True)
+
+    init_data: list[dict[str, NDArray]] = [
+        {
+            key: np.empty(len(table_fmt.rows), dtype="O")
+            for key in table_fmt.columns[table_idx]
+        }
+        for table_idx in table_fmt.preproc.idx_tables_size_verify
+    ]
+
+    for page_number in range(len(batch_image_gallery)):
+        output_path_page = output_path_book / str(page_number)
+        results = [
+            digitize.digitize_page_and_write_output(
+                book,
+                init_data,
+                page_number=page_number + 3,
+                date_str="2022-02-02",
+                model_path=model_path,
+                model_predict=ml.model_predict,
+                prob_thresh=0.5,
+                output_path_page=output_path_page,
+                output_text_fmt=True,
+                debug=True,
+            )
+        ]
+
+    collection = []
+    time.sleep(1)
+    gr.Info("Pages were succesfully digitized ✨")
+
+    yield collection, gr.skip()
+
+
+def all_example_images() -> list[str]:
+    """
+    Get paths to all example images.
+    """
+    examples = [
+        os.path.join(EXAMPLES_DIRECTORY, f"{pipeline}.png") for pipeline in PIPELINES
+    ]
+    return examples
+
+
+def get_selected_example_image(
+    first_page, last_page, event: gr.SelectData
+) -> tuple[str, io.Book] | None:
+    """
+    Get the name of the pipeline that corresponds to the selected image.
+    """
+    # for name, details in PIPELINES.items():
+    name, _ext = event.value["image"]["orig_name"].split(".")
+
+    if name in PIPELINES:
+        book_path = pooch.retrieve(**PIPELINES[name], path=DATA_CACHE)
+        first, last, book = io.read_book(book_path)
+        book._name = name
+        book.size_cell = [1.0, 1.0, 1.0, 1.0]
+        return [book.read_image(pg) for pg in range(first_page, last_page)], book
+
+
+table_fmt_config_override_placeholder = (
+    """\
+[default]
+version = 0
+
+# Default values, but wrote explicitly here. See PreprocConfig class
+[default.preproc]
+table_modif = true
+corr_rotate = true
+row_idx_unit = "HOURS"
+idx_tables_size_verify = [0, 1]
+
+[version.0]
+columns = [
+  [
+    "term_på_baro",
+    "barom",
+    "torra_term",
+    "våta_term",
+    "moln_slag_lägre",
+    "moln_mängd_lägre",
+    "moln_slag_medel",
+    "moln_slag_högre"
+  ],
+  [
+    "moln_het_sol_dimma_nederbörd_total",
+    "vind_riktning",
+    "vind_beaufort",
+    "vind_m_sek",
+    "sikt",
+    "sjögang",
+    "maximi_term",
+    "minimi_term",
+    "nederbörd_mängd",
+    "nederbörd_slag"
+  ]
+]
+name_idx = "tid"
+rows = [2, 8, 14, 19, 21]
+tables = [
+  [5, 8],
+  [5, 10],
+  [3, 1],
+  [4, 2],
+  [4, 5]
+]
+""",
+)
+
+with gr.Blocks() as submit:
+    gr.Markdown("# Upload")
+    gr.Markdown(
+        "Select or upload the image you want to transcribe. You can upload up to five images at a time."
+    )
+
+    batch_book_state = gr.State()
+    collection_submit_state = gr.State()
+
+    with gr.Group():
+        with gr.Row(equal_height=True):
+            with gr.Column(scale=5):
+                batch_image_gallery = gr.Gallery(
+                    file_types=["image"],
+                    label="Image to digitize",
+                    interactive=True,
+                    object_fit="scale-down",
+                    scale=10,
+                )
+
+            with gr.Column(scale=2):
+                first_page = gr.Number(3, label="First page of the book", precision=0)
+                last_page = gr.Number(4, label="Last page of the book", precision=0)
+                examples = gr.Gallery(
+                    all_example_images(),
+                    label="Examples",
+                    interactive=False,
+                    allow_preview=False,
+                    object_fit="scale-down",
+                    min_width=250,
+                )
+
+    with Modal(visible=False) as edit_table_fmt_modal:
+        with gr.Column():
+            gr.Markdown(
+                "## Table format configuration\n"
+                "Write a custom table format, overriding the default one. "
+                "Close [x] the popup when you are done."
+            )
+            table_fmt_config_override = gr.Code("", language="python")
+            gr.HTML(
+                (
+                    "<a href='https://dawsonia.readthedocs.io/en/latest/user_guide/misc.html#table-formats' target='_blank'>"
+                    "Read the docs for the table-formats spec"
+                    "</a>. "
+                ),
+                padding=False,
+                elem_classes="pipeline-help",
+            )
+
+    with gr.Row():
+        run_button = gr.Button("Digitize", variant="primary", scale=0, min_width=200)
+        edit_table_fmt_button = gr.Button(
+            "Edit table format", variant="secondary", scale=0, min_width=200
+        )
+
+    # All events interactions below
+
+    examples.select(
+        get_selected_example_image,
+        (first_page, last_page),
+        (batch_image_gallery, batch_book_state),
+    )
+
+    @batch_image_gallery.upload(
+        inputs=batch_image_gallery,
+        outputs=[batch_image_gallery],
+    )
+    def validate_images(images):
+        if len(images) > MAX_IMAGES:
+            gr.Warning(f"Maximum images you can upload is set to: {MAX_IMAGES}")
+            return gr.update(value=None)
+        return images
+
+    run_button.click(
+        fn=run_dawsonia,
+        inputs=[table_fmt_config_override, batch_image_gallery, batch_book_state],
+        outputs=[collection_submit_state, batch_image_gallery],
+    )
+    edit_table_fmt_button.click(lambda: Modal(visible=True), None, edit_table_fmt_modal)

pyproject.toml

@@ -6,7 +6,8 @@ readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
   "dawsonia[cuda]",
-  "htrflow>=0.2.2",
+  "gradio>=5.15.0",
+  "gradio-modal>=0.0.4",
   "pyarrow>=19.0.0",
   "typer<0.14",
 ]