Spaces:
Sleeping
Sleeping
File size: 2,704 Bytes
4c85050 417b347 4c85050 417b347 4c85050 417b347 4c85050 417b347 4c85050 417b347 4c85050 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
import os
import tarfile
import datasets
import pandas as pd
_CITATION = """\
@InProceedings{huggingface:dataset,
title = {Small htr examples images},
author={Gabriel Borg},
year={2023}
}
"""
_DESCRIPTION = """\
Demo dataset for the htr demo.
"""
_HOMEPAGE = "https://huggingface.co/datasets/Riksarkivet/test_images_demo"
_LICENSE = ""
_REPO = "https://huggingface.co/datasets/Riksarkivet/test_images_demo"
class ExampleImages(datasets.GeneratorBasedBuilder):
"""Small sample of image-text pairs"""
def _info(self):
return datasets.DatasetInfo(
description=_DESCRIPTION,
features=datasets.Features(
{
"text": datasets.Value("string"),
"image": datasets.Image(),
}
),
supervised_keys=None,
homepage=_HOMEPAGE,
citation=_CITATION,
)
def _split_generators(self, dl_manager):
images_archive = dl_manager.download(f"{_REPO}/resolve/main/images.tar.gz")
metadata_path = dl_manager.download(f"{_REPO}/resolve/main/images.txt")
image_iters = dl_manager.iter_archive(images_archive)
return [
datasets.SplitGenerator(
name=datasets.Split.TRAIN, gen_kwargs={"images": image_iters, "metadata_path": metadata_path}
),
]
def _generate_examples(self, images, metadata_path):
"""Generate images and text."""
with open(metadata_path, encoding="utf-8") as f:
metadata_list = f.read().split("\n")
for idx, (img_obj, meta_txt) in enumerate(zip(images, metadata_list)):
filepath, image = img_obj
text_value = meta_txt.split("= ")[-1].strip()
yield idx, {
"image": {"path": filepath, "bytes": image.read()},
"text": text_value,
}
def txt_to_csv(file_name):
text_file_path = f"{file_name}.txt"
df = pd.read_csv(text_file_path, delimiter="=", header=None, names=["Key", "Label"], encoding="utf-8")
df["Key"] = df["Key"].str.strip()
df["Label"] = df["Label"].str.strip()
print(df)
df.to_csv(f"{file_name}.csv", index=False)
def sort_and_compress_images(images_folder, tar_file):
sorted_images = sorted(os.listdir(images_folder))
with tarfile.open(tar_file, "w:gz") as tar:
for image_name in sorted_images:
image_path = os.path.join(images_folder, image_name)
tar.add(image_path, arcname=image_name)
print("Images sorted and compressed into tar.gz archive.")
if __name__ == "__main__":
txt_to_csv("info")
sort_and_compress_images("images", "sorted_images.tar.gz")
|