leverage infos file, fix prerun script

apputils.py

@@ -0,0 +1,15 @@
+from typing import Dict, List
+
+
+def new_state() -> Dict[str, List]:
+    return {
+        "task_categories": [],
+        "task_ids": [],
+        "multilinguality": [],
+        "languages": [],
+        "language_creators": [],
+        "annotations_creators": [],
+        "source_datasets": [],
+        "size_categories": [],
+        "licenses": [],
+    }

build_metadata_file.py

@@ -12,6 +12,8 @@ from typing import Dict
 
 import yaml
 
+from apputils import new_state
+
 
 def metadata_from_readme(f: Path) -> Dict:
     with f.open() as fi:
@@ -25,10 +27,10 @@ def metadata_from_readme(f: Path) -> Dict:
 def load_ds_datas():
     drepo = Path("datasets")
     if drepo.exists() and drepo.is_dir():
-        check_call(["git", "pull"], cwd=str((Path.cwd() / "datasets").absolute()))
+        check_call(["git", "pull"], cwd=drepo)
     else:
         check_call(["git", "clone", "https://github.com/huggingface/datasets.git"])
-    head_sha = check_output(["git", "rev-parse", "HEAD"])
+    head_sha = check_output(["git", "rev-parse", "HEAD"], cwd=drepo)
 
     datasets_md = dict()
 
@@ -38,6 +40,8 @@ def load_ds_datas():
             metadata = metadata_from_readme(ddir / "README.md")
         except:
             metadata = None
+        if metadata is None or len(metadata) == 0:
+            metadata = new_state()
 
         try:
             with (ddir / "dataset_infos.json").open() as fi:
@@ -45,8 +49,7 @@ def load_ds_datas():
         except:
             infos = None
 
-        if metadata is not None and len(metadata) > 0:
-            datasets_md[ddir.name] = dict(metadata=metadata, infos=infos)
+        datasets_md[ddir.name] = dict(metadata=metadata, infos=infos)
     return head_sha.decode().strip(), datasets_md
 
 

license_set.json

@@ -1,452 +0,0 @@
-{
-    "other": "Other license",
-    "unknown": "License information unavailable",
-    "0bsd": "BSD Zero Clause License",
-    "aal": "Attribution Assurance License",
-    "abstyles": "Abstyles License",
-    "adobe-2006": "Adobe Systems Incorporated Source Code License Agreement",
-    "adobe-glyph": "Adobe Glyph List License",
-    "adsl": "Amazon Digital Services License",
-    "afl-1.1": "Academic Free License v1.1",
-    "afl-1.2": "Academic Free License v1.2",
-    "afl-2.0": "Academic Free License v2.0",
-    "afl-2.1": "Academic Free License v2.1",
-    "afl-3.0": "Academic Free License v3.0",
-    "afmparse": "Afmparse License",
-    "agpl-1.0": "Affero General Public License v1.0",
-    "agpl-1.0-only": "Affero General Public License v1.0 only",
-    "agpl-1.0-or-later": "Affero General Public License v1.0 or later",
-    "agpl-3.0": "GNU Affero General Public License v3.0",
-    "agpl-3.0-only": "GNU Affero General Public License v3.0 only",
-    "agpl-3.0-or-later": "GNU Affero General Public License v3.0 or later",
-    "aladdin": "Aladdin Free Public License",
-    "amdplpa": "AMD's plpa_map.c License",
-    "aml": "Apple MIT License",
-    "ampas": "Academy of Motion Picture Arts and Sciences BSD",
-    "antlr-pd": "ANTLR Software Rights Notice",
-    "antlr-pd-fallback": "ANTLR Software Rights Notice with license fallback",
-    "apache-1.0": "Apache License 1.0",
-    "apache-1.1": "Apache License 1.1",
-    "apache-2.0": "Apache License 2.0",
-    "apafml": "Adobe Postscript AFM License",
-    "apl-1.0": "Adaptive Public License 1.0",
-    "apsl-1.0": "Apple Public Source License 1.0",
-    "apsl-1.1": "Apple Public Source License 1.1",
-    "apsl-1.2": "Apple Public Source License 1.2",
-    "apsl-2.0": "Apple Public Source License 2.0",
-    "artistic-1.0": "Artistic License 1.0",
-    "artistic-1.0-cl8": "Artistic License 1.0 w/clause 8",
-    "artistic-1.0-perl": "Artistic License 1.0 (Perl)",
-    "artistic-2.0": "Artistic License 2.0",
-    "bahyph": "Bahyph License",
-    "barr": "Barr License",
-    "beerware": "Beerware License",
-    "bittorrent-1.0": "BitTorrent Open Source License v1.0",
-    "bittorrent-1.1": "BitTorrent Open Source License v1.1",
-    "blessing": "SQLite Blessing",
-    "blueoak-1.0.0": "Blue Oak Model License 1.0.0",
-    "borceux": "Borceux license",
-    "bsd-1-clause": "BSD 1-Clause License",
-    "bsd-2-clause": "BSD 2-Clause \"Simplified\" License",
-    "bsd-2-clause-freebsd": "BSD 2-Clause FreeBSD License",
-    "bsd-2-clause-netbsd": "BSD 2-Clause NetBSD License",
-    "bsd-2-clause-patent": "BSD-2-Clause Plus Patent License",
-    "bsd-2-clause-views": "BSD 2-Clause with views sentence",
-    "bsd-3-clause": "BSD 3-Clause \"New\" or \"Revised\" License",
-    "bsd-3-clause-attribution": "BSD with attribution",
-    "bsd-3-clause-clear": "BSD 3-Clause Clear License",
-    "bsd-3-clause-lbnl": "Lawrence Berkeley National Labs BSD variant license",
-    "bsd-3-clause-no-nuclear-license": "BSD 3-Clause No Nuclear License",
-    "bsd-3-clause-no-nuclear-license-2014": "BSD 3-Clause No Nuclear License 2014",
-    "bsd-3-clause-no-nuclear-warranty": "BSD 3-Clause No Nuclear Warranty",
-    "bsd-3-clause-open-mpi": "BSD 3-Clause Open MPI variant",
-    "bsd-4-clause": "BSD 4-Clause \"Original\" or \"Old\" License",
-    "bsd-4-clause-uc": "BSD-4-Clause (University of California-Specific)",
-    "bsd-protection": "BSD Protection License",
-    "bsd-source-code": "BSD Source Code Attribution",
-    "bsl-1.0": "Boost Software License 1.0",
-    "busl-1.1": "Business Source License 1.1",
-    "bzip2-1.0.5": "bzip2 and libbzip2 License v1.0.5",
-    "bzip2-1.0.6": "bzip2 and libbzip2 License v1.0.6",
-    "cal-1.0": "Cryptographic Autonomy License 1.0",
-    "cal-1.0-combined-work-exception": "Cryptographic Autonomy License 1.0 (Combined Work Exception)",
-    "caldera": "Caldera License",
-    "catosl-1.1": "Computer Associates Trusted Open Source License 1.1",
-    "cc-by-1.0": "Creative Commons Attribution 1.0 Generic",
-    "cc-by-2.0": "Creative Commons Attribution 2.0 Generic",
-    "cc-by-2.5": "Creative Commons Attribution 2.5 Generic",
-    "cc-by-3.0": "Creative Commons Attribution 3.0 Unported",
-    "cc-by-3.0-at": "Creative Commons Attribution 3.0 Austria",
-    "cc-by-3.0-us": "Creative Commons Attribution 3.0 United States",
-    "cc-by-4.0": "Creative Commons Attribution 4.0 International",
-    "cc-by-nc-1.0": "Creative Commons Attribution Non Commercial 1.0 Generic",
-    "cc-by-nc-2.0": "Creative Commons Attribution Non Commercial 2.0 Generic",
-    "cc-by-nc-2.5": "Creative Commons Attribution Non Commercial 2.5 Generic",
-    "cc-by-nc-3.0": "Creative Commons Attribution Non Commercial 3.0 Unported",
-    "cc-by-nc-4.0": "Creative Commons Attribution Non Commercial 4.0 International",
-    "cc-by-nc-nd-1.0": "Creative Commons Attribution Non Commercial No Derivatives 1.0 Generic",
-    "cc-by-nc-nd-2.0": "Creative Commons Attribution Non Commercial No Derivatives 2.0 Generic",
-    "cc-by-nc-nd-2.5": "Creative Commons Attribution Non Commercial No Derivatives 2.5 Generic",
-    "cc-by-nc-nd-3.0": "Creative Commons Attribution Non Commercial No Derivatives 3.0 Unported",
-    "cc-by-nc-nd-3.0-igo": "Creative Commons Attribution Non Commercial No Derivatives 3.0 IGO",
-    "cc-by-nc-nd-4.0": "Creative Commons Attribution Non Commercial No Derivatives 4.0 International",
-    "cc-by-nc-sa-1.0": "Creative Commons Attribution Non Commercial Share Alike 1.0 Generic",
-    "cc-by-nc-sa-2.0": "Creative Commons Attribution Non Commercial Share Alike 2.0 Generic",
-    "cc-by-nc-sa-2.5": "Creative Commons Attribution Non Commercial Share Alike 2.5 Generic",
-    "cc-by-nc-sa-3.0": "Creative Commons Attribution Non Commercial Share Alike 3.0 Unported",
-    "cc-by-nc-sa-4.0": "Creative Commons Attribution Non Commercial Share Alike 4.0 International",
-    "cc-by-nd-1.0": "Creative Commons Attribution No Derivatives 1.0 Generic",
-    "cc-by-nd-2.0": "Creative Commons Attribution No Derivatives 2.0 Generic",
-    "cc-by-nd-2.5": "Creative Commons Attribution No Derivatives 2.5 Generic",
-    "cc-by-nd-3.0": "Creative Commons Attribution No Derivatives 3.0 Unported",
-    "cc-by-nd-4.0": "Creative Commons Attribution No Derivatives 4.0 International",
-    "cc-by-sa-1.0": "Creative Commons Attribution Share Alike 1.0 Generic",
-    "cc-by-sa-2.0": "Creative Commons Attribution Share Alike 2.0 Generic",
-    "cc-by-sa-2.0-uk": "Creative Commons Attribution Share Alike 2.0 England and Wales",
-    "cc-by-sa-2.5": "Creative Commons Attribution Share Alike 2.5 Generic",
-    "cc-by-sa-3.0": "Creative Commons Attribution Share Alike 3.0 Unported",
-    "cc-by-sa-3.0-at": "Creative Commons Attribution-Share Alike 3.0 Austria",
-    "cc-by-sa-4.0": "Creative Commons Attribution Share Alike 4.0 International",
-    "cc-pddc": "Creative Commons Public Domain Dedication and Certification",
-    "cc0-1.0": "Creative Commons Zero v1.0 Universal",
-    "cddl-1.0": "Common Development and Distribution License 1.0",
-    "cddl-1.1": "Common Development and Distribution License 1.1",
-    "cdla-permissive-1.0": "Community Data License Agreement Permissive 1.0",
-    "cdla-sharing-1.0": "Community Data License Agreement Sharing 1.0",
-    "cecill-1.0": "CeCILL Free Software License Agreement v1.0",
-    "cecill-1.1": "CeCILL Free Software License Agreement v1.1",
-    "cecill-2.0": "CeCILL Free Software License Agreement v2.0",
-    "cecill-2.1": "CeCILL Free Software License Agreement v2.1",
-    "cecill-b": "CeCILL-B Free Software License Agreement",
-    "cecill-c": "CeCILL-C Free Software License Agreement",
-    "cern-ohl-1.1": "CERN Open Hardware Licence v1.1",
-    "cern-ohl-1.2": "CERN Open Hardware Licence v1.2",
-    "cern-ohl-p-2.0": "CERN Open Hardware Licence Version 2 - Permissive",
-    "cern-ohl-s-2.0": "CERN Open Hardware Licence Version 2 - Strongly Reciprocal",
-    "cern-ohl-w-2.0": "CERN Open Hardware Licence Version 2 - Weakly Reciprocal",
-    "clartistic": "Clarified Artistic License",
-    "cnri-jython": "CNRI Jython License",
-    "cnri-python": "CNRI Python License",
-    "cnri-python-gpl-compatible": "CNRI Python Open Source GPL Compatible License Agreement",
-    "condor-1.1": "Condor Public License v1.1",
-    "copyleft-next-0.3.0": "copyleft-next 0.3.0",
-    "copyleft-next-0.3.1": "copyleft-next 0.3.1",
-    "cpal-1.0": "Common Public Attribution License 1.0",
-    "cpl-1.0": "Common Public License 1.0",
-    "cpol-1.02": "Code Project Open License 1.02",
-    "crossword": "Crossword License",
-    "crystalstacker": "CrystalStacker License",
-    "cua-opl-1.0": "CUA Office Public License v1.0",
-    "cube": "Cube License",
-    "curl": "curl License",
-    "d-fsl-1.0": "Deutsche Freie Software Lizenz",
-    "diffmark": "diffmark license",
-    "doc": "DOC License",
-    "dotseqn": "Dotseqn License",
-    "dsdp": "DSDP License",
-    "dvipdfm": "dvipdfm License",
-    "ecl-1.0": "Educational Community License v1.0",
-    "ecl-2.0": "Educational Community License v2.0",
-    "ecos-2.0": "eCos license version 2.0",
-    "efl-1.0": "Eiffel Forum License v1.0",
-    "efl-2.0": "Eiffel Forum License v2.0",
-    "egenix": "eGenix.com Public License 1.1.0",
-    "entessa": "Entessa Public License v1.0",
-    "epics": "EPICS Open License",
-    "epl-1.0": "Eclipse Public License 1.0",
-    "epl-2.0": "Eclipse Public License 2.0",
-    "erlpl-1.1": "Erlang Public License v1.1",
-    "etalab-2.0": "Etalab Open License 2.0",
-    "eudatagrid": "EU DataGrid Software License",
-    "eupl-1.0": "European Union Public License 1.0",
-    "eupl-1.1": "European Union Public License 1.1",
-    "eupl-1.2": "European Union Public License 1.2",
-    "eurosym": "Eurosym License",
-    "fair": "Fair License",
-    "frameworx-1.0": "Frameworx Open License 1.0",
-    "freeimage": "FreeImage Public License v1.0",
-    "fsfap": "FSF All Permissive License",
-    "fsful": "FSF Unlimited License",
-    "fsfullr": "FSF Unlimited License (with License Retention)",
-    "ftl": "Freetype Project License",
-    "gfdl-1.1": "GNU Free Documentation License v1.1",
-    "gfdl-1.1-invariants-only": "GNU Free Documentation License v1.1 only - invariants",
-    "gfdl-1.1-invariants-or-later": "GNU Free Documentation License v1.1 or later - invariants",
-    "gfdl-1.1-no-invariants-only": "GNU Free Documentation License v1.1 only - no invariants",
-    "gfdl-1.1-no-invariants-or-later": "GNU Free Documentation License v1.1 or later - no invariants",
-    "gfdl-1.1-only": "GNU Free Documentation License v1.1 only",
-    "gfdl-1.1-or-later": "GNU Free Documentation License v1.1 or later",
-    "gfdl-1.2": "GNU Free Documentation License v1.2",
-    "gfdl-1.2-invariants-only": "GNU Free Documentation License v1.2 only - invariants",
-    "gfdl-1.2-invariants-or-later": "GNU Free Documentation License v1.2 or later - invariants",
-    "gfdl-1.2-no-invariants-only": "GNU Free Documentation License v1.2 only - no invariants",
-    "gfdl-1.2-no-invariants-or-later": "GNU Free Documentation License v1.2 or later - no invariants",
-    "gfdl-1.2-only": "GNU Free Documentation License v1.2 only",
-    "gfdl-1.2-or-later": "GNU Free Documentation License v1.2 or later",
-    "gfdl-1.3": "GNU Free Documentation License v1.3",
-    "gfdl-1.3-invariants-only": "GNU Free Documentation License v1.3 only - invariants",
-    "gfdl-1.3-invariants-or-later": "GNU Free Documentation License v1.3 or later - invariants",
-    "gfdl-1.3-no-invariants-only": "GNU Free Documentation License v1.3 only - no invariants",
-    "gfdl-1.3-no-invariants-or-later": "GNU Free Documentation License v1.3 or later - no invariants",
-    "gfdl-1.3-only": "GNU Free Documentation License v1.3 only",
-    "gfdl-1.3-or-later": "GNU Free Documentation License v1.3 or later",
-    "giftware": "Giftware License",
-    "gl2ps": "GL2PS License",
-    "glide": "3dfx Glide License",
-    "glulxe": "Glulxe License",
-    "glwtpl": "Good Luck With That Public License",
-    "gnuplot": "gnuplot License",
-    "gpl-1.0": "GNU General Public License v1.0 only",
-    "gpl-1.0+": "GNU General Public License v1.0 or later",
-    "gpl-1.0-only": "GNU General Public License v1.0 only",
-    "gpl-1.0-or-later": "GNU General Public License v1.0 or later",
-    "gpl-2.0": "GNU General Public License v2.0 only",
-    "gpl-2.0+": "GNU General Public License v2.0 or later",
-    "gpl-2.0-only": "GNU General Public License v2.0 only",
-    "gpl-2.0-or-later": "GNU General Public License v2.0 or later",
-    "gpl-2.0-with-autoconf-exception": "GNU General Public License v2.0 w/Autoconf exception",
-    "gpl-2.0-with-bison-exception": "GNU General Public License v2.0 w/Bison exception",
-    "gpl-2.0-with-classpath-exception": "GNU General Public License v2.0 w/Classpath exception",
-    "gpl-2.0-with-font-exception": "GNU General Public License v2.0 w/Font exception",
-    "gpl-2.0-with-gcc-exception": "GNU General Public License v2.0 w/GCC Runtime Library exception",
-    "gpl-3.0": "GNU General Public License v3.0 only",
-    "gpl-3.0+": "GNU General Public License v3.0 or later",
-    "gpl-3.0-only": "GNU General Public License v3.0 only",
-    "gpl-3.0-or-later": "GNU General Public License v3.0 or later",
-    "gpl-3.0-with-autoconf-exception": "GNU General Public License v3.0 w/Autoconf exception",
-    "gpl-3.0-with-gcc-exception": "GNU General Public License v3.0 w/GCC Runtime Library exception",
-    "gsoap-1.3b": "gSOAP Public License v1.3b",
-    "haskellreport": "Haskell Language Report License",
-    "hippocratic-2.1": "Hippocratic License 2.1",
-    "hpnd": "Historical Permission Notice and Disclaimer",
-    "hpnd-sell-variant": "Historical Permission Notice and Disclaimer - sell variant",
-    "htmltidy": "HTML Tidy License",
-    "ibm-pibs": "IBM PowerPC Initialization and Boot Software",
-    "icu": "ICU License",
-    "ijg": "Independent JPEG Group License",
-    "imagemagick": "ImageMagick License",
-    "imatix": "iMatix Standard Function Library Agreement",
-    "imlib2": "Imlib2 License",
-    "info-zip": "Info-ZIP License",
-    "intel": "Intel Open Source License",
-    "intel-acpi": "Intel ACPI Software License Agreement",
-    "interbase-1.0": "Interbase Public License v1.0",
-    "ipa": "IPA Font License",
-    "ipl-1.0": "IBM Public License v1.0",
-    "isc": "ISC License",
-    "jasper-2.0": "JasPer License",
-    "jpnic": "Japan Network Information Center License",
-    "json": "JSON License",
-    "lal-1.2": "Licence Art Libre 1.2",
-    "lal-1.3": "Licence Art Libre 1.3",
-    "latex2e": "Latex2e License",
-    "leptonica": "Leptonica License",
-    "lgpl-2.0": "GNU Library General Public License v2 only",
-    "lgpl-2.0+": "GNU Library General Public License v2 or later",
-    "lgpl-2.0-only": "GNU Library General Public License v2 only",
-    "lgpl-2.0-or-later": "GNU Library General Public License v2 or later",
-    "lgpl-2.1": "GNU Lesser General Public License v2.1 only",
-    "lgpl-2.1+": "GNU Library General Public License v2.1 or later",
-    "lgpl-2.1-only": "GNU Lesser General Public License v2.1 only",
-    "lgpl-2.1-or-later": "GNU Lesser General Public License v2.1 or later",
-    "lgpl-3.0": "GNU Lesser General Public License v3.0 only",
-    "lgpl-3.0+": "GNU Lesser General Public License v3.0 or later",
-    "lgpl-3.0-only": "GNU Lesser General Public License v3.0 only",
-    "lgpl-3.0-or-later": "GNU Lesser General Public License v3.0 or later",
-    "lgpllr": "Lesser General Public License For Linguistic Resources",
-    "libpng": "libpng License",
-    "libpng-2.0": "PNG Reference Library version 2",
-    "libselinux-1.0": "libselinux public domain notice",
-    "libtiff": "libtiff License",
-    "liliq-p-1.1": "Licence Libre du Qu\u00e9bec \u2013 Permissive version 1.1",
-    "liliq-r-1.1": "Licence Libre du Qu\u00e9bec \u2013 R\u00e9ciprocit\u00e9 version 1.1",
-    "liliq-rplus-1.1": "Licence Libre du Qu\u00e9bec \u2013 R\u00e9ciprocit\u00e9 forte version 1.1",
-    "linux-openib": "Linux Kernel Variant of OpenIB.org license",
-    "lpl-1.0": "Lucent Public License Version 1.0",
-    "lpl-1.02": "Lucent Public License v1.02",
-    "lppl-1.0": "LaTeX Project Public License v1.0",
-    "lppl-1.1": "LaTeX Project Public License v1.1",
-    "lppl-1.2": "LaTeX Project Public License v1.2",
-    "lppl-1.3a": "LaTeX Project Public License v1.3a",
-    "lppl-1.3c": "LaTeX Project Public License v1.3c",
-    "makeindex": "MakeIndex License",
-    "miros": "The MirOS Licence",
-    "mit": "MIT License",
-    "mit-0": "MIT No Attribution",
-    "mit-advertising": "Enlightenment License (e16)",
-    "mit-cmu": "CMU License",
-    "mit-enna": "enna License",
-    "mit-feh": "feh License",
-    "mit-open-group": "MIT Open Group variant",
-    "mitnfa": "MIT +no-false-attribs license",
-    "motosoto": "Motosoto License",
-    "mpich2": "mpich2 License",
-    "mpl-1.0": "Mozilla Public License 1.0",
-    "mpl-1.1": "Mozilla Public License 1.1",
-    "mpl-2.0": "Mozilla Public License 2.0",
-    "mpl-2.0-no-copyleft-exception": "Mozilla Public License 2.0 (no copyleft exception)",
-    "ms-pl": "Microsoft Public License",
-    "ms-rl": "Microsoft Reciprocal License",
-    "mtll": "Matrix Template Library License",
-    "mulanpsl-1.0": "Mulan Permissive Software License, Version 1",
-    "mulanpsl-2.0": "Mulan Permissive Software License, Version 2",
-    "multics": "Multics License",
-    "mup": "Mup License",
-    "nasa-1.3": "NASA Open Source Agreement 1.3",
-    "naumen": "Naumen Public License",
-    "nbpl-1.0": "Net Boolean Public License v1",
-    "ncgl-uk-2.0": "Non-Commercial Government Licence",
-    "ncsa": "University of Illinois/NCSA Open Source License",
-    "net-snmp": "Net-SNMP License",
-    "netcdf": "NetCDF license",
-    "newsletr": "Newsletr License",
-    "ngpl": "Nethack General Public License",
-    "nist-pd": "NIST Public Domain Notice",
-    "nist-pd-fallback": "NIST Public Domain Notice with license fallback",
-    "nlod-1.0": "Norwegian Licence for Open Government Data",
-    "nlpl": "No Limit Public License",
-    "nokia": "Nokia Open Source License",
-    "nosl": "Netizen Open Source License",
-    "noweb": "Noweb License",
-    "npl-1.0": "Netscape Public License v1.0",
-    "npl-1.1": "Netscape Public License v1.1",
-    "nposl-3.0": "Non-Profit Open Software License 3.0",
-    "nrl": "NRL License",
-    "ntp": "NTP License",
-    "ntp-0": "NTP No Attribution",
-    "nunit": "Nunit License",
-    "o-uda-1.0": "Open Use of Data Agreement v1.0",
-    "occt-pl": "Open CASCADE Technology Public License",
-    "oclc-2.0": "OCLC Research Public License 2.0",
-    "odbl-1.0": "ODC Open Database License v1.0",
-    "odc-by-1.0": "Open Data Commons Attribution License v1.0",
-    "ofl-1.0": "SIL Open Font License 1.0",
-    "ofl-1.0-no-rfn": "SIL Open Font License 1.0 with no Reserved Font Name",
-    "ofl-1.0-rfn": "SIL Open Font License 1.0 with Reserved Font Name",
-    "ofl-1.1": "SIL Open Font License 1.1",
-    "ofl-1.1-no-rfn": "SIL Open Font License 1.1 with no Reserved Font Name",
-    "ofl-1.1-rfn": "SIL Open Font License 1.1 with Reserved Font Name",
-    "ogc-1.0": "OGC Software License, Version 1.0",
-    "ogl-canada-2.0": "Open Government Licence - Canada",
-    "ogl-uk-1.0": "Open Government Licence v1.0",
-    "ogl-uk-2.0": "Open Government Licence v2.0",
-    "ogl-uk-3.0": "Open Government Licence v3.0",
-    "ogtsl": "Open Group Test Suite License",
-    "oldap-1.1": "Open LDAP Public License v1.1",
-    "oldap-1.2": "Open LDAP Public License v1.2",
-    "oldap-1.3": "Open LDAP Public License v1.3",
-    "oldap-1.4": "Open LDAP Public License v1.4",
-    "oldap-2.0": "Open LDAP Public License v2.0 (or possibly 2.0A and 2.0B)",
-    "oldap-2.0.1": "Open LDAP Public License v2.0.1",
-    "oldap-2.1": "Open LDAP Public License v2.1",
-    "oldap-2.2": "Open LDAP Public License v2.2",
-    "oldap-2.2.1": "Open LDAP Public License v2.2.1",
-    "oldap-2.2.2": "Open LDAP Public License 2.2.2",
-    "oldap-2.3": "Open LDAP Public License v2.3",
-    "oldap-2.4": "Open LDAP Public License v2.4",
-    "oldap-2.5": "Open LDAP Public License v2.5",
-    "oldap-2.6": "Open LDAP Public License v2.6",
-    "oldap-2.7": "Open LDAP Public License v2.7",
-    "oldap-2.8": "Open LDAP Public License v2.8",
-    "oml": "Open Market License",
-    "openssl": "OpenSSL License",
-    "opl-1.0": "Open Public License v1.0",
-    "oset-pl-2.1": "OSET Public License version 2.1",
-    "osl-1.0": "Open Software License 1.0",
-    "osl-1.1": "Open Software License 1.1",
-    "osl-2.0": "Open Software License 2.0",
-    "osl-2.1": "Open Software License 2.1",
-    "osl-3.0": "Open Software License 3.0",
-    "parity-6.0.0": "The Parity Public License 6.0.0",
-    "parity-7.0.0": "The Parity Public License 7.0.0",
-    "pddl-1.0": "ODC Public Domain Dedication & License 1.0",
-    "php-3.0": "PHP License v3.0",
-    "php-3.01": "PHP License v3.01",
-    "plexus": "Plexus Classworlds License",
-    "polyform-noncommercial-1.0.0": "PolyForm Noncommercial License 1.0.0",
-    "polyform-small-business-1.0.0": "PolyForm Small Business License 1.0.0",
-    "postgresql": "PostgreSQL License",
-    "psf-2.0": "Python Software Foundation License 2.0",
-    "psfrag": "psfrag License",
-    "psutils": "psutils License",
-    "python-2.0": "Python License 2.0",
-    "qhull": "Qhull License",
-    "qpl-1.0": "Q Public License 1.0",
-    "rdisc": "Rdisc License",
-    "rhecos-1.1": "Red Hat eCos Public License v1.1",
-    "rpl-1.1": "Reciprocal Public License 1.1",
-    "rpl-1.5": "Reciprocal Public License 1.5",
-    "rpsl-1.0": "RealNetworks Public Source License v1.0",
-    "rsa-md": "RSA Message-Digest License",
-    "rscpl": "Ricoh Source Code Public License",
-    "ruby": "Ruby License",
-    "sax-pd": "Sax Public Domain Notice",
-    "saxpath": "Saxpath License",
-    "scea": "SCEA Shared Source License",
-    "sendmail": "Sendmail License",
-    "sendmail-8.23": "Sendmail License 8.23",
-    "sgi-b-1.0": "SGI Free Software License B v1.0",
-    "sgi-b-1.1": "SGI Free Software License B v1.1",
-    "sgi-b-2.0": "SGI Free Software License B v2.0",
-    "shl-0.5": "Solderpad Hardware License v0.5",
-    "shl-0.51": "Solderpad Hardware License, Version 0.51",
-    "simpl-2.0": "Simple Public License 2.0",
-    "sissl": "Sun Industry Standards Source License v1.1",
-    "sissl-1.2": "Sun Industry Standards Source License v1.2",
-    "sleepycat": "Sleepycat License",
-    "smlnj": "Standard ML of New Jersey License",
-    "smppl": "Secure Messaging Protocol Public License",
-    "snia": "SNIA Public License 1.1",
-    "spencer-86": "Spencer License 86",
-    "spencer-94": "Spencer License 94",
-    "spencer-99": "Spencer License 99",
-    "spl-1.0": "Sun Public License v1.0",
-    "ssh-openssh": "SSH OpenSSH license",
-    "ssh-short": "SSH short notice",
-    "sspl-1.0": "Server Side Public License, v 1",
-    "standardml-nj": "Standard ML of New Jersey License",
-    "sugarcrm-1.1.3": "SugarCRM Public License v1.1.3",
-    "swl": "Scheme Widget Library (SWL) Software License Agreement",
-    "tapr-ohl-1.0": "TAPR Open Hardware License v1.0",
-    "tcl": "TCL/TK License",
-    "tcp-wrappers": "TCP Wrappers License",
-    "tmate": "TMate Open Source License",
-    "torque-1.1": "TORQUE v2.5+ Software License v1.1",
-    "tosl": "Trusster Open Source License",
-    "tu-berlin-1.0": "Technische Universitaet Berlin License 1.0",
-    "tu-berlin-2.0": "Technische Universitaet Berlin License 2.0",
-    "ucl-1.0": "Upstream Compatibility License v1.0",
-    "unicode-dfs-2015": "Unicode License Agreement - Data Files and Software (2015)",
-    "unicode-dfs-2016": "Unicode License Agreement - Data Files and Software (2016)",
-    "unicode-tou": "Unicode Terms of Use",
-    "unlicense": "The Unlicense",
-    "upl-1.0": "Universal Permissive License v1.0",
-    "vim": "Vim License",
-    "vostrom": "VOSTROM Public License for Open Source",
-    "vsl-1.0": "Vovida Software License v1.0",
-    "w3c": "W3C Software Notice and License (2002-12-31)",
-    "w3c-19980720": "W3C Software Notice and License (1998-07-20)",
-    "w3c-20150513": "W3C Software Notice and Document License (2015-05-13)",
-    "watcom-1.0": "Sybase Open Watcom Public License 1.0",
-    "wsuipa": "Wsuipa License",
-    "wtfpl": "Do What The F*ck You Want To Public License",
-    "wxwindows": "wxWindows Library License",
-    "x11": "X11 License",
-    "xerox": "Xerox License",
-    "xfree86-1.1": "XFree86 License 1.1",
-    "xinetd": "xinetd License",
-    "xnet": "X.Net License",
-    "xpp": "XPP License",
-    "xskat": "XSkat License",
-    "ypl-1.0": "Yahoo! Public License v1.0",
-    "ypl-1.1": "Yahoo! Public License v1.1",
-    "zed": "Zed License",
-    "zend-2.0": "Zend License v2.0",
-    "zimbra-1.3": "Zimbra Public License v1.3",
-    "zimbra-1.4": "Zimbra Public License v1.4",
-    "zlib": "zlib License",
-    "zlib-acknowledgement": "zlib/libpng License with Acknowledgement",
-    "zpl-1.1": "Zope Public License 1.1",
-    "zpl-2.0": "Zope Public License 2.0",
-    "zpl-2.1": "Zope Public License 2.1"
-}

tagging_app.py

@@ -1,4 +1,5 @@
 import json
+import logging
 from pathlib import Path
 from typing import Callable, Dict, List, Tuple
 
@@ -14,6 +15,8 @@ from datasets.utils.metadata import (
     known_task_ids,
 )
 
+from apputils import new_state
+
 st.set_page_config(
     page_title="HF Dataset Tagging App",
     page_icon="https://huggingface.co/front/assets/huggingface_logo.svg",
@@ -38,7 +41,7 @@ st.markdown(
 ########################
 
 
-def load_ds_datas():
+def load_ds_datas() -> Dict[str, Dict[str, Dict]]:
     metada_exports = sorted(
         [f for f in Path.cwd().iterdir() if f.name.startswith("metadata_")],
         key=lambda f: f.lstat().st_mtime,
@@ -47,6 +50,7 @@ def load_ds_datas():
     if len(metada_exports) == 0:
         raise ValueError("need to run ./build_metada_file.py at least once")
     with metada_exports[0].open() as fi:
+        logging.info(f"loaded {metada_exports[0]}")
         return json.load(fi)
 
 
@@ -81,18 +85,32 @@ def validate_dict(w: st.delta_generator.DeltaGenerator, state_dict: Dict):
         w.error(e)
 
 
-def new_state() -> Dict[str, List]:
-    return {
-        "task_categories": [],
-        "task_ids": [],
-        "multilinguality": [],
-        "languages": [],
-        "language_creators": [],
-        "annotations_creators": [],
-        "source_datasets": [],
-        "size_categories": [],
-        "licenses": [],
-    }
+def map_num_examples_to_size_categories(n: int) -> str:
+    if n <= 0:
+        size_cat = "unknown"
+    elif n < 1000:
+        size_cat = "n<1K"
+    elif n < 10000:
+        size_cat = "1K<n<10K"
+    elif n < 100000:
+        size_cat = "10K<n<100K"
+    elif n < 1000000:
+        size_cat = "100K<n<1M"
+    elif n < 10000000:
+        size_cat = "1M<n<10M"
+    elif n < 100000000:
+        size_cat = "10M<n<100M"
+    elif n < 1000000000:
+        size_cat = "100M<n<1B"
+    elif n < 10000000000:
+        size_cat = "1B<n<10B"
+    elif n < 100000000000:
+        size_cat = "10B<n<100B"
+    elif n < 1000000000000:
+        size_cat = "100B<n<1T"
+    else:
+        size_cat = "n>1T"
+    return size_cat
 
 
 def is_state_empty(state: Dict[str, List]) -> bool:
@@ -101,8 +119,9 @@ def is_state_empty(state: Dict[str, List]) -> bool:
 
 state = new_state()
 datasets_md = load_ds_datas()
-existing_tag_sets = {name: mds["metadata"] for name, mds in datasets_md.items()}
-all_dataset_ids = list(existing_tag_sets.keys())
+dataset_ids = list(datasets_md.keys())
+dataset_id_to_metadata = {name: mds["metadata"] for name, mds in datasets_md.items()}
+dataset_id_to_infos = {name: mds["infos"] for name, mds in datasets_md.items()}
 
 
 ########################
@@ -124,19 +143,26 @@ queryparams = st.experimental_get_query_params()
 preload = queryparams.get("preload_dataset", list())
 preloaded_id = None
 initial_state = None
-did_index = 0
-if len(preload) == 1 and preload[0] in all_dataset_ids:
+initial_infos, initial_info_cfg = None, None
+dataset_selector_index = 0
+
+if len(preload) == 1 and preload[0] in dataset_ids:
     preloaded_id, *_ = preload
-    initial_state = existing_tag_sets.get(preloaded_id)
+    initial_state = dataset_id_to_metadata.get(preloaded_id)
+    initial_infos = dataset_id_to_infos.get(preloaded_id)
+    initial_info_cfg = next(iter(initial_infos)) if initial_infos is not None else None  # pick first available config
     state = initial_state or new_state()
-    did_index = all_dataset_ids.index(preloaded_id)
+    dataset_selector_index = dataset_ids.index(preloaded_id)
 
 preloaded_id = st.sidebar.selectbox(
-    label="Choose dataset to load tag set from", options=all_dataset_ids, index=did_index
+    label="Choose dataset to load tag set from", options=dataset_ids, index=dataset_selector_index
 )
+
 leftbtn, rightbtn = st.sidebar.beta_columns(2)
 if leftbtn.button("pre-load"):
-    initial_state = existing_tag_sets[preloaded_id]
+    initial_state = dataset_id_to_metadata[preloaded_id]
+    initial_infos = dataset_id_to_infos[preloaded_id]
+    initial_info_cfg = next(iter(initial_infos))  # pick first available config
     state = initial_state or new_state()
     st.experimental_set_query_params(preload_dataset=preloaded_id)
 if not is_state_empty(state):
@@ -168,6 +194,9 @@ Here is the matching yaml block:
 leftcol, _, rightcol = st.beta_columns([12, 1, 12])
 
 
+#
+# TASKS
+#
 leftcol.markdown("### Supported tasks")
 state["task_categories"] = multiselect(
     leftcol,
@@ -197,6 +226,9 @@ for task_category in state["task_categories"]:
 state["task_ids"] = task_specifics
 
 
+#
+# LANGUAGES
+#
 leftcol.markdown("### Languages")
 state["multilinguality"] = multiselect(
     leftcol,
@@ -233,6 +265,10 @@ langtags = leftcol.text_area(
 )
 state["languages"] = langtags.split(";")
 
+
+#
+# DATASET CREATORS & ORIGINS
+#
 leftcol.markdown("### Dataset creators")
 state["language_creators"] = multiselect(
     leftcol,
@@ -250,6 +286,9 @@ state["annotations_creators"] = multiselect(
 )
 
 
+#
+# LICENSES
+#
 state["licenses"] = multiselect(
     leftcol,
     "Licenses",
@@ -266,7 +305,10 @@ if "other" in state["licenses"]:
     st.write(f"Registering other-{other_license} license")
     state["licenses"][state["licenses"].index("other")] = f"other-{other_license}"
 
-# link to supported datasets
+
+#
+# LINK TO SUPPORTED DATASETS
+#
 pre_select_ext_a = []
 if "original" in state["source_datasets"]:
     pre_select_ext_a += ["original"]
@@ -288,7 +330,7 @@ if "extended" in state["extended"]:
         "Linked datasets",
         "Which other datasets does this one use data from?",
         values=pre_select_ext_b,
-        valid_set=all_dataset_ids + ["other"],
+        valid_set=dataset_ids + ["other"],
     )
     if "other" in extended_sources:
         other_extended_sources = leftcol.text_input(
@@ -299,17 +341,23 @@ if "extended" in state["extended"]:
         extended_sources[extended_sources.index("other")] = f"other-{other_extended_sources}"
     state["source_datasets"] += [f"extended|{src}" for src in extended_sources]
 
+
+#
+# SIZE CATEGORY
+#
+leftcol.markdown("### Size category")
+logging.info(initial_infos[initial_info_cfg]["splits"] if initial_infos is not None else 0)
+initial_num_examples = (
+    sum([dct.get("num_examples", 0) for _split, dct in initial_infos[initial_info_cfg].get("splits", dict()).items()])
+    if initial_infos is not None
+    else -1
+)
+initial_size_cats = map_num_examples_to_size_categories(initial_num_examples)
+leftcol.markdown(f"Computed size category from automatically generated dataset info to: `{initial_size_cats}`")
 current_size_cats = state.get("size_categories") or ["unknown"]
 ok, nonok = split_known(current_size_cats, known_size_categories)
 if len(nonok) > 0:
     leftcol.markdown(f"**Found bad codes in existing tagset**:\n{nonok}")
-state["size_categories"] = [
-    leftcol.selectbox(
-        "What is the size category of the dataset?",
-        options=known_size_categories,
-        index=known_size_categories.index(ok[0]) if len(ok) > 0 else 0,
-    )
-]
 
 
 ########################

task_set.json

@@ -1,86 +0,0 @@
-{
-    "conditional-text-generation": {
-        "description": "data-to-text and text transduction tasks such as translation or summarization",
-        "options": [
-            "machine-translation",
-            "sentence-splitting-fusion",
-            "summarization",
-            "table-to-text",
-            "text-simplification",
-            "explanation-generation",
-            "other-stuctured-to-text",
-            "other"
-        ]
-    },
-    "question-answering": {
-        "description": "question answering tasks",
-        "options": [
-            "open-domain-qa",
-            "closed-domain-qa",
-            "multiple-choice-qa",
-            "extractive-qa",
-            "abstractive-qa",
-            "other"
-        ]
-    },
-    "sequence-modeling": {
-        "description": "such as language modeling or dialogue",
-        "options": [
-            "dialogue-modeling",
-            "language-modeling",
-            "other-multi-turn",
-            "slot-filling",
-            "other"
-        ]
-    },
-    "structure-prediction": {
-        "description": "predicting structural properties of the text, such as syntax",
-        "options": [
-            "coreference-resolution",
-            "named-entity-recognition",
-            "part-of-speech-tagging",
-            "parsing",
-            "other"
-        ]
-    },
-    "text-classification": {
-        "description": "predicting a class index or boolean value",
-        "options": [
-            "acceptability-classification",
-            "entity-linking-classification",
-            "fact-checking",
-            "intent-classification",
-            "multi-class-classification",
-            "multi-label-classification",
-            "natural-language-inference",
-            "semantic-similarity-classification",
-            "sentiment-classification",
-            "topic-classification",
-            "other"
-        ]
-    },
-    "text-retrieval": {
-        "description": "information or text retrieval tasks",
-        "options": [
-            "document-retrieval",
-            "utterance-retrieval",
-            "entity-linking-retrieval",
-            "fact-checking-retrieval",
-            "other"
-        ]
-    },
-    "text-scoring": {
-        "description": "text scoring tasks, predicting a real valued score for some text",
-        "options": [
-            "semantic-similarity-scoring",
-            "sentiment-scoring",
-            "other"
-        ]
-    },
-    "other": {
-        "description": "other task family not mentioned here",
-        "options": [
-            "other"
-        ]
-    }
-}