Merge pull request #1 from jeff52415/release/v1.0.0

.dockerignore

@@ -0,0 +1,20 @@
+.git
+.gitignore
+.env
+.venv
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.Python
+*.py[cod]
+*$py.class
+.pytest_cache
+.coverage
+htmlcov
+.mypy_cache
+.ruff_cache
+.DS_Store
+notebooks/
+tests/
+docs/

.github/workflows/ci.yml

@@ -0,0 +1,75 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install Poetry
+        run: |
+          pipx install poetry
+
+      - name: Configure Poetry
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+
+      - name: Cache Poetry virtualenv
+        uses: actions/cache@v3
+        with:
+          path: ./.venv
+          key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        run: |
+          poetry install
+
+      - name: Run pre-commit hooks
+        uses: pre-commit/[email protected]
+        with:
+          extra_args: --all-files
+        env:
+          PRE_COMMIT_CACHE_KEY: ${{ hashFiles('.pre-commit-config.yaml', 'pyproject.toml') }}
+
+      - name: Run tests with coverage
+        run: |
+          poetry run pytest --cov=llmdataparser --cov-report=xml
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          fail_ci_if_error: true
+
+      - name: Build documentation
+        run: |
+          poetry add mkdocs mkdocs-material --group dev
+          cp README.md docs/index.md
+          poetry run mkdocs build
+        if: matrix.python-version == '3.12'
+
+      - name: Upload documentation artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: documentation
+          path: site/
+        if: matrix.python-version == '3.12'

.github/workflows/docker.yml

@@ -0,0 +1,43 @@
+name: Docker CD
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "Dockerfile"
+      - ".dockerignore"
+      - "docker-compose.yml"
+      - "pyproject.toml"
+      - "poetry.lock"
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Get version from pyproject.toml
+        run: |
+          echo "VERSION=$(grep '^version = ' pyproject.toml | cut -d'"' -f2)" >> $GITHUB_ENV
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          push: true
+          tags: |
+            jeff52415/llmdataparser:latest
+            jeff52415/llmdataparser:v${{ env.VERSION }}
+          cache-from: type=registry,ref=jeff52415/llmdataparser:latest
+          cache-to: type=inline
+          platforms: linux/amd64,linux/arm64

.github/workflows/huggingface-sync.yml

@@ -0,0 +1,30 @@
+name: Deploy to Hugging Face Space
+
+on:
+  push:
+    branches: [main]
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Configure Git
+        run: |
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "github-actions[bot]"
+
+      - name: Login to Hugging Face
+        env:
+          HF_TOKEN: ${{ secrets.HUGGINGFACE_TOKEN }}
+        run: |
+          huggingface-cli login --token $HF_TOKEN --add-to-git-credential
+
+      - name: Push to Hugging Face Space
+        run: |
+          git remote add space https://huggingface.co/spaces/JeffYang52415/LLMEval-Dataset-Parser
+          git push space main:main

.gitignore

@@ -8,8 +8,6 @@ build/
 dist/
 *.egg-info/
 
-# Poetry
-poetry.lock
 
 # Virtual environment
 .env/
@@ -32,3 +30,15 @@ poetry.lock
 
 # Mac files
 .DS_Store
+
+#gradio cache
+.cache/
+.gradio/
+
+#notebook cache
+.ipynb_checkpoints/
+notebooks/
+
+#coverage
+.coverage
+.coverage.*

.pre-commit-config.yaml

@@ -1,18 +1,12 @@
 # .pre-commit-config.yaml
 
 repos:
-  - repo: https://github.com/psf/black
-    rev: 23.9.1
-    hooks:
-      - id: black
-        args: ["--target-version=py311"]
-        additional_dependencies: ["typing-extensions>=4.8.0"]
-  - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.4.4
     hooks:
-      - id: flake8
-        additional_dependencies: ["typing-extensions>=4.8.0"]
-        args: ["--ignore=E203, E501, W503, E501"]
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
   - repo: https://github.com/PyCQA/isort
     rev: 5.12.0
     hooks:
@@ -24,10 +18,10 @@ repos:
       - id: mypy
         args:
           [
-            "--python-version=3.11",
+            "--config-file=pyproject.toml",
             "--install-types",
             "--non-interactive",
-            "--ignore-missing-imports",
+            "--exclude=tests/*",
           ]
         additional_dependencies:
           - "typing-extensions>=4.8.0"
@@ -41,7 +35,6 @@ repos:
       - id: detect-aws-credentials
         args: ["--allow-missing-credentials"]
       - id: detect-private-key
-      - id: end-of-file-fixer
       - id: check-added-large-files
       - id: check-ast
       - id: check-byte-order-marker
@@ -49,9 +42,6 @@ repos:
       - id: check-docstring-first
       - id: check-json
       - id: debug-statements
-      - id: detect-private-key
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
       - id: mixed-line-ending
   - repo: https://github.com/myint/autoflake
     rev: v2.2.1
@@ -70,16 +60,36 @@ repos:
     hooks:
       - id: prettier
         types_or: [markdown, yaml]
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.4.4
-    hooks:
-      # Run the linter.
-      - id: ruff
-        args: [--fix]
-      # Run the formatter.
-      - id: ruff-format
   - repo: https://github.com/kynan/nbstripout
     rev: 0.5.0 # use the latest version
     hooks:
       - id: nbstripout
+  - repo: https://github.com/nbQA-dev/nbQA
+    rev: 1.7.1
+    hooks:
+      - id: nbqa-ruff
+        additional_dependencies: [ruff]
+      - id: nbqa-isort
+      - id: nbqa-flake8
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.15.0
+    hooks:
+      - id: pyupgrade
+  - repo: https://github.com/executablebooks/mdformat
+    rev: 0.7.17
+    hooks:
+      - id: mdformat
+        additional_dependencies:
+          - mdformat-gfm # GitHub-flavored Markdown
+          - mdformat-frontmatter # YAML frontmatter
+          - mdformat-footnote
+  - repo: https://github.com/shellcheck-py/shellcheck-py
+    rev: v0.9.0.6
+    hooks:
+      - id: shellcheck
+  - repo: https://github.com/pycqa/bandit
+    rev: 1.7.7
+    hooks:
+      - id: bandit
+        args: ["-c", "pyproject.toml"]
+        additional_dependencies: ["bandit[toml]", ".[toml]"]

CHANGELOG.md

@@ -0,0 +1,16 @@
+# Changelog
+
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## \[1.0.0\] - 2024-12-30
+
+### Added
+
+- Initial release
+- Support for multiple benchmark datasets (MMLU, GSM8k, etc.)
+- Gradio interface for dataset exploration
+- Comprehensive test suite
+- Documentation and examples

Dockerfile

@@ -0,0 +1,69 @@
+# Use Python 3.12 slim image as base
+FROM python:3.12-slim
+
+# Set environment variables
+ENV PYTHONUNBUFFERED=1 \
+    POETRY_VERSION=1.7.1 \
+    POETRY_HOME="/opt/poetry" \
+    POETRY_NO_INTERACTION=1 \
+    GRADIO_SERVER_NAME=0.0.0.0 \
+    GRADIO_SERVER_PORT=7860
+
+# Set working directory
+WORKDIR /app
+
+# Create cache directories for Hugging Face and set permissions correctly
+ENV HF_HOME=/app/.cache/huggingface
+RUN mkdir -p /app/.cache/huggingface && \
+    mkdir -p /app/.cache/torch && \
+    mkdir -p /app/.cache/transformers
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y \
+    portaudio19-dev \
+    python3-pip \
+    gcc \
+    git \
+    curl \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean
+
+# Install poetry
+RUN pip install poetry==${POETRY_VERSION} && \
+    poetry config virtualenvs.create false
+
+# Copy dependency files first
+COPY pyproject.toml poetry.lock ./
+
+# Install dependencies using the lock file
+RUN poetry install --no-dev --no-interaction --no-ansi
+
+# Create app user and group with specific UID/GID
+RUN groupadd -r app --gid 1000 && \
+    useradd -r -g app --uid 1000 --create-home app
+
+# Set ownership of all cache directories
+RUN chown -R app:app /app/.cache && \
+    chmod -R 755 /app/.cache
+
+# Before switching to non-root user, create and set permissions
+RUN mkdir -p /home/app/.cache && \
+    mkdir -p /home/app/.config/matplotlib && \
+    chown -R app:app /home/app/.cache && \
+    chown -R app:app /home/app/.config
+
+# Set matplotlib config dir
+ENV MPLCONFIGDIR=/home/app/.config/matplotlib
+
+# Switch to non-root user
+USER app
+
+# Copy the rest of the application
+COPY --chown=app:app . .
+
+# Expose the port the app runs on
+EXPOSE 7860
+
+# Run the application
+ENTRYPOINT ["python"]
+CMD ["app.py"]

Makefile

@@ -0,0 +1,107 @@
+# -----------------------------
+# Variables
+# -----------------------------
+IMAGE_NAME = llmdataparser
+CONTAINER_NAME = llmdataparser
+VERSION = latest
+
+# -----------------------------
+# Docker Basic Commands
+# -----------------------------
+# Build the Docker image
+build:
+	docker build -t $(IMAGE_NAME):$(VERSION) .
+
+# Run the container
+run:
+	docker run -d -p 7860:7860 --name $(CONTAINER_NAME) $(IMAGE_NAME):$(VERSION)
+
+# Stop the container
+stop:
+	docker stop $(CONTAINER_NAME)
+
+# Remove the container
+rm:
+	docker rm $(CONTAINER_NAME)
+
+# Remove the image
+rmi:
+	docker rmi $(IMAGE_NAME):$(VERSION)
+
+# -----------------------------
+# Docker Compose Commands
+# -----------------------------
+# Start with docker-compose (development)
+compose-up:
+	docker compose up -d
+
+# Stop and remove containers
+compose-down:
+	docker compose down
+
+# View logs
+compose-logs:
+	docker compose logs -f
+
+# Rebuild containers
+compose-build:
+	docker compose build
+
+# Restart containers
+compose-restart:
+	docker compose restart
+
+# -----------------------------
+# Convenience Commands
+# -----------------------------
+# Build and run with docker
+up: build run
+
+# Stop and remove container
+down: stop rm
+
+# Clean everything
+clean: stop rm rmi
+
+# -----------------------------
+# Monitoring Commands
+# -----------------------------
+# Show container logs
+logs:
+	docker logs $(CONTAINER_NAME)
+
+# Follow container logs
+logs-follow:
+	docker logs -f $(CONTAINER_NAME)
+
+# Show container status
+status:
+	docker ps -a | grep $(CONTAINER_NAME)
+
+# Enter container shell
+shell:
+	docker exec -it $(CONTAINER_NAME) /bin/bash
+
+# -----------------------------
+# Production Commands
+# -----------------------------
+# Test nginx configuration (for production use)
+nginx-test:
+	docker compose run --rm nginx nginx -t
+
+# Start with nginx test (for production use)
+compose-up-prod: nginx-test compose-up
+
+# -----------------------------
+# Security Commands
+# -----------------------------
+security-check:
+	@echo "Checking nginx configuration..."
+	docker compose run --rm nginx nginx -t
+	@echo "Checking exposed ports..."
+	docker compose config | grep -E "ports:|127.0.0.1"
+
+# Ensure all targets are treated as commands, not files
+.PHONY: build run stop rm rmi clean up down logs shell \
+        compose-up compose-down compose-logs compose-build compose-restart \
+        nginx-test status logs-follow compose-up-prod

README.md

@@ -1,6 +1,17 @@
+---
+title: LLMEval Dataset Parser
+emoji: ⚡
+colorFrom: green
+colorTo: gray
+sdk: docker
+pinned: false
+license: mit
+short_description: A collection of parsers for LLM benchmark datasets
+---
+
 # LLMDataParser
 
-**LLMDataParser** is a Python library that provides parsers for benchmark datasets used in evaluating Large Language Models (LLMs). It offers a unified interface for loading and parsing datasets like **MMLU** and **GSM8k**, simplifying dataset preparation for LLM evaluation.
+**LLMDataParser** is a Python library that provides parsers for benchmark datasets used in evaluating Large Language Models (LLMs). It offers a unified interface for loading and parsing datasets like **MMLU**, **GSM8k**, and others, streamlining dataset preparation for LLM evaluation. The library aims to simplify the process of working with common LLM benchmark datasets through a consistent API.
 
 ## Features
 
@@ -8,6 +19,7 @@
 - **LLM-Agnostic**: Independent of any specific language model.
 - **Easy to Use**: Simple methods and built-in Python types.
 - **Extensible**: Easily add support for new datasets.
+- **Gradio**: Built-in Gradio interface for interactive dataset exploration and testing.
 
 ## Installation
 
@@ -22,7 +34,7 @@ You can install the package directly using `pip`. Even with only a `pyproject.to
    cd LLMDataParser
    ```
 
-2. **Install Dependencies with pip**:
+1. **Install Dependencies with pip**:
 
    ```bash
    pip install .
@@ -38,7 +50,7 @@ Poetry manages the virtual environment and dependencies automatically, so you do
    poetry install
    ```
 
-2. **Activate the Virtual Environment**:
+1. **Activate the Virtual Environment**:
 
    ```bash
    poetry shell
@@ -46,7 +58,58 @@ Poetry manages the virtual environment and dependencies automatically, so you do
 
 ## Available Parsers
 
-- **MMLUDatasetParser**: Parses the MMLU dataset.
+- **MMLUDatasetParser**
+- **MMLUProDatasetParser**
+- **MMLUReduxDatasetParser**
+- **TMMLUPlusDatasetParser**
+- **GSM8KDatasetParser**
+- **MATHDatasetParser**
+- **MGSMDatasetParser**
+- **HumanEvalDatasetParser**
+- **HumanEvalDatasetPlusParser**
+- **BBHDatasetParser**
+- **MBPPDatasetParser**
+- **IFEvalDatasetParser**
+- **TWLegalDatasetParser**
+- **TMLUDatasetParser**
+
+## Quick Start Guide
+
+Here's a simple example demonstrating how to use the library:
+
+```python
+ from llmdataparser import ParserRegistry
+ # list all available parsers
+ ParserRegistry.list_parsers()
+ # get a parser
+ parser = ParserRegistry.get_parser("mmlu")
+ # load the parser
+ parser.load() # optional: task_name, split
+ # parse the parser
+ parser.parse() # optional: split_names
+
+ print(parser.task_names)
+ print(parser.split_names)
+ print(parser.get_dataset_description)
+ print(parser.get_huggingface_link)
+ print(parser.total_tasks)
+ data = parser.get_parsed_data
+```
+
+We also provide a Gradio demo for interactive testing:
+
+```bash
+python app.py
+```
+
+## Adding New Dataset Parsers
+
+To add support for a new dataset, please refer to our detailed guide in [docs/adding_new_parser.md](docs/adding_new_parser.md). The guide includes:
+
+- Step-by-step instructions for creating a new parser
+- Code examples and templates
+- Best practices and common patterns
+- Testing guidelines
 
 ## License
 

app.py

@@ -0,0 +1,456 @@
+import secrets
+from functools import lru_cache
+from typing import Any
+
+import gradio as gr
+
+from llmdataparser import ParserRegistry
+from llmdataparser.base_parser import (
+    VALID_CATEGORIES,
+    DatasetDescription,
+    DatasetParser,
+    EvaluationMetric,
+    ParseEntry,
+)
+
+
+@lru_cache(maxsize=32)
+def get_parser_instance(parser_name: str) -> DatasetParser[Any]:
+    """Get a cached parser instance by name."""
+    return ParserRegistry.get_parser(parser_name)
+
+
+def get_available_splits(parser: DatasetParser[Any]) -> list[str] | None:
+    """Get available splits for the selected parser after loading."""
+    if not hasattr(parser, "split_names") or not parser.split_names:
+        return None
+    return list(parser.split_names)
+
+
+def get_available_tasks(parser: DatasetParser[Any]) -> list[str]:
+    """Get available tasks for the selected parser."""
+    if not hasattr(parser, "task_names"):
+        return ["default"]
+    return list(parser.task_names)
+
+
+def format_entry_attributes(entry: ParseEntry) -> str:
+    """Format all attributes of a ParseEntry except question and answer."""
+    from dataclasses import fields
+
+    # Get all field names from the dataclass
+    field_names = [field.name for field in fields(entry)]
+    # Filter out question and answer
+    filtered_fields = [
+        name for name in field_names if name not in ["question", "answer"]
+    ]
+    # Build the formatted string
+    return "\n".join(f"{name}: {getattr(entry, name)}" for name in filtered_fields)
+
+
+def load_and_parse(
+    parser_name: str, task_name: str | None, split_name: str | None
+) -> tuple[int, str, str, str, gr.Dropdown, str]:
+    """Load and parse the dataset, return the first entry and available splits."""
+    try:
+        parser = get_parser_instance(parser_name)
+
+        # Load the dataset
+        parser.load(
+            task_name=task_name if task_name != "default" else None,
+            split=split_name,
+            trust_remote_code=True,
+        )
+
+        # Get available splits after loading
+        available_splits = get_available_splits(parser)
+
+        # Parse the dataset
+        parser.parse(split_names=split_name, force=True)
+
+        # Get parsed data
+        parsed_data = parser.get_parsed_data
+
+        split_dropdown = gr.Dropdown(
+            choices=available_splits,
+            label="Select Split",
+            interactive=True,
+            value=None,
+            allow_custom_value=True,
+        )
+
+        info = parser.__repr__()
+        if not parsed_data:
+            return 0, "", "", "", split_dropdown, info
+
+        # Get the first entry
+        first_entry = parsed_data[0]
+
+        return (
+            0,  # Return first index instead of list of indices
+            first_entry.question,
+            first_entry.answer,
+            format_entry_attributes(first_entry),
+            split_dropdown,
+            info,
+        )
+    except Exception as e:
+        # Make the error message more user-friendly and detailed
+        error_msg = f"Failed to load dataset: {str(e)}\nParser: {parser_name}\nTask: {task_name}\nSplit: {split_name}"
+        return 0, error_msg, "", "", [], ""
+
+
+def update_entry(
+    parsed_data_index: int | None, parser_name: str
+) -> tuple[str, str, str]:
+    """Update the displayed entry based on the selected index."""
+    try:
+        if not parser_name:
+            return "Please select a parser first", "", ""
+
+        parser = get_parser_instance(parser_name)
+        parsed_data = parser.get_parsed_data
+
+        if not parsed_data:
+            return "No data available", "", ""
+
+        if parsed_data_index is None:
+            # Random selection using secrets instead of random
+            random_index = secrets.randbelow(len(parsed_data))
+            entry = parsed_data[random_index]
+        else:
+            # Ensure index is within bounds
+            index = max(0, min(parsed_data_index, len(parsed_data) - 1))
+            entry = parsed_data[index]
+
+        return (
+            entry.question,
+            entry.answer,
+            format_entry_attributes(entry),
+        )
+    except Exception as e:
+        return f"Error: {str(e)}", "", ""
+
+
+def update_parser_options(parser_name: str) -> tuple[gr.Dropdown, gr.Dropdown, str]:
+    """Update available tasks and splits for the selected parser."""
+    try:
+        parser = get_parser_instance(parser_name)
+        tasks = get_available_tasks(parser)
+        default_task = getattr(parser, "_default_task", "default")
+
+        # Update task dropdown
+        task_dropdown = gr.Dropdown(
+            choices=tasks,
+            value=default_task,
+            label="Select Task",
+            interactive=True,
+            allow_custom_value=True,
+        )
+
+        # Update split dropdown - Note the value is now explicitly None
+        splits = get_available_splits(parser)
+        split_dropdown = gr.Dropdown(
+            choices=splits,
+            label="Select Split",
+            interactive=True,
+            value=None,
+            allow_custom_value=True,
+        )
+
+        info = parser.__repr__()
+        return task_dropdown, split_dropdown, info
+    except Exception as e:
+        return (
+            gr.Dropdown(choices=["default"], value="default"),
+            gr.Dropdown(choices=[]),
+            f"Error: {str(e)}",
+        )
+
+
+def clear_parser_cache() -> None:
+    """Clear the parser cache."""
+    get_parser_instance.cache_clear()
+
+
+def format_dataset_description(description: DatasetDescription) -> str:
+    """Format DatasetDescription into a readable string."""
+    formatted = [
+        f"# {description.name}",
+        f"\n**Purpose**: {description.purpose}",
+        f"\n**Language**: {description.language}",
+        f"\n**Format**: {description.format}",
+        f"\n**Source**: {description.source}",
+        f"\n**Characteristics**: {description.characteristics}",
+    ]
+
+    if description.citation:
+        formatted.append(f"\n**Citation**:\n```\n{description.citation}\n```")
+
+    if description.additional_info:
+        formatted.append("\n**Additional Information**:")
+        for key, value in description.additional_info.items():
+            formatted.append(f"- {key}: {value}")
+
+    return "\n".join(formatted)
+
+
+def get_primary_metrics(metrics: list[EvaluationMetric]) -> list[str]:
+    """Get list of primary metric names."""
+    return [metric.name for metric in metrics if metric.primary]
+
+
+def format_metric_details(metric: EvaluationMetric) -> str:
+    """Format a single EvaluationMetric into a readable string."""
+    return f"""# {metric.name}<br>
+                **Type**: {metric.type}<br>
+                **Description**: {metric.description}"""
+
+
+def update_dataset_info(parser_name: str) -> tuple:
+    """Update dataset description and evaluation metrics information."""
+    try:
+        parser = get_parser_instance(parser_name)
+        description = parser.get_dataset_description()
+        metrics = parser.get_evaluation_metrics()
+
+        # Format description
+        desc_text = format_dataset_description(description)
+
+        # Get primary metrics for dropdown
+        primary_metrics = get_primary_metrics(metrics)
+
+        # Format details for first metric (or empty if no metrics)
+        first_metric = metrics[0] if metrics else None
+        metric_details = format_metric_details(first_metric) if first_metric else ""
+
+        return (
+            gr.Markdown(value=desc_text),
+            gr.Dropdown(
+                choices=primary_metrics,
+                value=primary_metrics[0] if primary_metrics else None,
+            ),
+            gr.Markdown(value=metric_details),
+        )
+    except Exception as e:
+        return (
+            gr.Markdown(value=f"Error loading dataset description: {str(e)}"),
+            gr.Dropdown(choices=[]),
+            gr.Markdown(value=""),
+        )
+
+
+def update_metric_details(metric_name: str, parser_name: str) -> str:
+    """Update the displayed metric details when selection changes."""
+    try:
+        parser = get_parser_instance(parser_name)
+        metrics = parser.get_evaluation_metrics()
+        selected_metric = next((m for m in metrics if m.name == metric_name), None)
+        return format_metric_details(selected_metric) if selected_metric else ""
+    except Exception as e:
+        return f"Error loading metric details: {str(e)}"
+
+
+def get_parser_categories(parser_name: str) -> list[str]:
+    """Get categories for a specific parser."""
+    try:
+        parser = get_parser_instance(parser_name)
+        description = parser.get_dataset_description()
+        return description.category
+    except Exception:
+        return []
+
+
+def filter_parsers_by_category(category: str | None) -> list[str]:
+    """Filter available parsers by category."""
+    if not category:
+        return ParserRegistry.list_parsers()
+
+    filtered_parsers = []
+    for parser_name in ParserRegistry.list_parsers():
+        categories = get_parser_categories(parser_name)
+        if category in categories:
+            filtered_parsers.append(parser_name)
+    return filtered_parsers
+
+
+def create_interface() -> gr.Blocks:
+    """Create and return the Gradio interface."""
+    with gr.Blocks(css="footer {display: none !important}") as demo:
+        # Add header section with purpose and GitHub info
+        gr.Markdown("""
+            # LLM Evaluation Dataset Parser
+
+            ### 🎯 Purpose
+            A unified interface for parsing and exploring various LLM benchmark datasets (MMLU, MMLU-Pro, GSM8k, and more).
+            This tool helps researchers and developers to:
+            - Easily explore different benchmark datasets
+            - Access standardized parsing for multiple dataset formats
+            - View dataset descriptions and evaluation metrics
+
+            ### 🔗 Links
+            - [GitHub Repository](https://github.com/jeff52415/LLMDataParser)
+            - [Documentation](https://github.com/jeff52415/LLMDataParser#readme)
+
+            ---
+        """)
+
+        # State management
+        parser_state = gr.State("")
+        dataset_status = gr.Textbox(label="Dataset Status", interactive=False)
+
+        with gr.Tabs():
+            with gr.Tab("Dataset Explorer"):
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        # Add category dropdown before parser selection
+                        category_dropdown = gr.Dropdown(
+                            choices=["All"] + list(VALID_CATEGORIES),
+                            label="Filter by Category",
+                            value="All",
+                            interactive=True,
+                        )
+
+                        # Parser selection and controls
+                        available_parsers = ParserRegistry.list_parsers()
+                        parser_dropdown = gr.Dropdown(
+                            choices=available_parsers,
+                            label="Select Parser",
+                            value=available_parsers[0] if available_parsers else None,
+                            interactive=True,
+                            allow_custom_value=True,
+                        )
+                        task_dropdown = gr.Dropdown(
+                            choices=["default"],
+                            label="Select Task",
+                            value="default",
+                            interactive=True,
+                            allow_custom_value=True,
+                        )
+                        split_dropdown = gr.Dropdown(
+                            choices=[],
+                            label="Select Split",
+                            interactive=True,
+                            value=None,
+                            allow_custom_value=True,
+                        )
+                        load_button = gr.Button(
+                            "Load and Parse Dataset", variant="primary"
+                        )
+
+                        # Entry selection
+                        entry_index = gr.Number(
+                            label="Select Entry Index (empty for random)",
+                            precision=0,
+                            interactive=True,
+                        )
+                        update_button = gr.Button(
+                            "Update/Random Entry", variant="secondary"
+                        )
+
+                    with gr.Column(scale=2):
+                        # Output displays
+                        question_output = gr.Textbox(
+                            label="Question", lines=5, show_copy_button=True
+                        )
+                        answer_output = gr.Textbox(
+                            label="Answer", lines=5, show_copy_button=True
+                        )
+                        attributes_output = gr.Textbox(
+                            label="Other Attributes", lines=5, show_copy_button=True
+                        )
+
+            with gr.Tab("Dataset Information"):
+                with gr.Row():
+                    with gr.Column(scale=2):
+                        # Dataset description
+                        dataset_description = gr.Markdown()
+
+                    with gr.Column(scale=1):
+                        # Evaluation metrics
+                        gr.Markdown("## Evaluation Metrics")
+                        metric_dropdown = gr.Dropdown(
+                            label="Select Primary Metric", interactive=True
+                        )
+                        metric_details = gr.Markdown()
+
+        # Add new event handler for category filtering
+        def update_parser_list(category: str) -> gr.Dropdown:
+            filtered_parsers = filter_parsers_by_category(
+                None if category == "All" else category
+            )
+            return gr.Dropdown(
+                choices=filtered_parsers,
+                value=filtered_parsers[0] if filtered_parsers else None,
+            )
+
+        category_dropdown.change(
+            fn=update_parser_list, inputs=[category_dropdown], outputs=[parser_dropdown]
+        )
+
+        # Event handlers
+        parser_dropdown.change(
+            fn=update_parser_options,
+            inputs=parser_dropdown,
+            outputs=[
+                task_dropdown,
+                split_dropdown,
+                dataset_status,
+            ],
+        ).then(lambda x: x, inputs=parser_dropdown, outputs=parser_state).then(
+            fn=update_dataset_info,
+            inputs=[parser_dropdown],
+            outputs=[dataset_description, metric_dropdown, metric_details],
+        )
+
+        load_button.click(
+            fn=load_and_parse,
+            inputs=[parser_dropdown, task_dropdown, split_dropdown],
+            outputs=[
+                entry_index,
+                question_output,
+                answer_output,
+                attributes_output,
+                split_dropdown,
+                dataset_status,
+            ],
+            api_name="load_and_parse",
+            show_progress="full",
+        ).then(
+            fn=update_dataset_info,
+            inputs=[parser_dropdown],
+            outputs=[dataset_description, metric_dropdown, metric_details],
+        )
+
+        update_button.click(
+            fn=update_entry,
+            inputs=[entry_index, parser_state],
+            outputs=[
+                question_output,
+                answer_output,
+                attributes_output,
+            ],
+            api_name="update_entry",
+        )
+
+        metric_dropdown.change(
+            fn=update_metric_details,
+            inputs=[metric_dropdown, parser_dropdown],
+            outputs=metric_details,
+        )
+
+    return demo
+
+
+if __name__ == "__main__":
+    print("Starting Gradio interface...")  # Add debug logging
+    demo = create_interface()
+    try:
+        demo.launch(
+            show_error=True,  # Changed to True for debugging
+        )
+    except Exception as e:
+        print(f"Error launching Gradio: {e}")  # Add error logging
+        import traceback
+
+        traceback.print_exc()

docker-compose.yml

@@ -0,0 +1,34 @@
+version: "3.8"
+
+services:
+  llmdataparser:
+    build: jeff52415/llmdataparser
+    environment:
+      - GRADIO_SERVER_PORT=7860
+    volumes:
+      - .:/app
+      - huggingface_cache:/app/.cache/huggingface
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://127.0.0.1:7860"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+    networks:
+      - internal
+
+  nginx:
+    image: nginx:alpine
+    ports:
+      - "80:80"
+    volumes:
+      - ./nginx.conf:/etc/nginx/nginx.conf:ro
+    depends_on:
+      - llmdataparser
+    networks:
+      - internal
+
+networks:
+  internal:
+
+volumes:
+  huggingface_cache:

docs/POETRY_JUPYTER_SETUP.md

@@ -1,51 +0,0 @@
-# Connecting Poetry Environment with Jupyter Notebook
-
-This guide provides simple steps to connect a Poetry-managed environment to Jupyter Notebook.
-
-## Steps
-
-1. **Activate the Poetry Environment**
-
-   First, navigate to your project directory and activate the Poetry shell:
-
-   ```bash
-   poetry shell
-   ```
-
-2. **Install Jupyter as a Development Dependency**
-
-   If Jupyter is not already installed, add it as a development dependency:
-
-   ```bash
-   poetry add --group dev jupyter
-   ```
-
-3. **Register the Poetry Environment as a Jupyter Kernel**
-
-   Run this command to make the Poetry environment available as a Jupyter kernel:
-
-   ```bash
-   python -m ipykernel install --user --name=llmdataparser-env --display-name "Python (LLMDataParser)"
-   ```
-
-   - `--name=llmdataparser-env`: Assigns a name to the kernel.
-   - `--display-name "Python (LLMDataParser)"`: Sets the display name seen in Jupyter.
-
-4. **Start Jupyter Notebook**
-
-   Launch Jupyter Notebook from the Poetry shell:
-
-   ```bash
-   jupyter notebook
-   ```
-
-5. **Select the Poetry Kernel in Jupyter**
-
-   - Open a notebook in Jupyter.
-   - Go to "Kernel" > "Change kernel" and select **Python (LLMDataParser)** from the list.
-
-   This connects the notebook to your Poetry environment.
-
----
-
-You’re now set up to use your Poetry environment within Jupyter Notebook!

docs/POETRY_USAGE.md

@@ -1,158 +0,0 @@
-# Poetry Usage Guide
-
-This guide provides instructions on how to use [Poetry](https://python-poetry.org/) to manage dependencies, install packages, and prepare your project for both development and production environments.
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Installing Poetry](#installing-poetry)
-- [Using Poetry in Development](#using-poetry-in-development)
-  - [Installing Dependencies](#installing-dependencies)
-  - [Updating Dependencies](#updating-dependencies)
-  - [Adding and Removing Dependencies](#adding-and-removing-dependencies)
-  - [Synchronizing Dependencies](#synchronizing-dependencies)
-- [Using Poetry in Production](#using-poetry-in-production)
-  - [Locking Dependencies](#locking-dependencies)
-  - [Installing from `poetry.lock`](#installing-from-poetrylock)
-- [Poetry Commands Summary](#poetry-commands-summary)
-
----
-
-## Overview
-
-Poetry is a dependency manager and build tool for Python projects. It simplifies managing dependencies, creating virtual environments, and ensuring version consistency between development and production environments. Poetry relies on two files:
-
-- **`pyproject.toml`**: Defines the dependencies and configuration.
-- **`poetry.lock`**: Locks dependencies to specific versions to ensure consistency.
-
----
-
-## Installing Poetry(macOS only)
-
-To install Poetry, use the following command:
-
-```bash
-brew install poetry
-```
-
-Refer to the [Poetry documentation](https://python-poetry.org/docs/#installation) for more options and OS-specific installation instructions.
-
----
-
-## Using Poetry in Development
-
-### Installing Dependencies
-
-In development, install dependencies specified in `pyproject.toml`:
-
-1. Navigate to the project directory:
-
-   ```bash
-   cd path/to/project
-   ```
-
-2. Run:
-   ```bash
-   poetry install
-   ```
-
-This command creates a virtual environment, installs all dependencies, and ensures they are compatible with the Python version specified.
-
-### Updating Dependencies
-
-During development, you can update dependencies by editing `pyproject.toml` directly and then running:
-
-```bash
-poetry install
-```
-
-This will apply any changes and update the environment without manually adding each dependency.
-
-### Adding and Removing Dependencies
-
-- **Add a New Dependency**:
-
-  ```bash
-  poetry add <package-name>
-  ```
-
-  Example:
-
-  ```bash
-  poetry add requests
-  ```
-
-- **Add a Development Dependency** (only used for development/testing):
-
-  ```bash
-  poetry add --group dev <package-name>
-  ```
-
-  Example:
-
-  ```bash
-  poetry add --group dev pytest
-  ```
-
-- **Remove a Dependency**:
-  ```bash
-  poetry remove <package-name>
-  ```
-
-### Synchronizing Dependencies
-
-If the `pyproject.toml` or `poetry.lock` files are updated (e.g., after pulling changes), run:
-
-```bash
-poetry install
-```
-
-This keeps your environment synchronized with any updates made to the dependency files.
-
----
-
-## Using Poetry in Production
-
-### Locking Dependencies
-
-To lock dependencies for production use, run:
-
-```bash
-poetry lock
-```
-
-This creates or updates `poetry.lock`, which pins each dependency to a specific version. This lock file should be used to maintain consistency in production.
-
-### Installing from `poetry.lock`
-
-In production, use `poetry.lock` to ensure exact dependency versions:
-
-1. Install only the required (non-development) dependencies:
-   ```bash
-   poetry install --no-dev
-   ```
-
-This ensures that dependencies are installed exactly as defined in `poetry.lock`.
-
----
-
-## Poetry Commands Summary
-
-| Command                        | Description                                                   |
-| ------------------------------ | ------------------------------------------------------------- |
-| `poetry install`               | Installs dependencies from `pyproject.toml` or `poetry.lock`. |
-| `poetry add <package-name>`    | Adds a new dependency and updates `pyproject.toml`.           |
-| `poetry add --group dev <pkg>` | Adds a development-only dependency.                           |
-| `poetry remove <package-name>` | Removes a dependency and updates `pyproject.toml`.            |
-| `poetry update`                | Updates all dependencies to their latest compatible versions. |
-| `poetry lock`                  | Locks dependencies to specific versions for production.       |
-| `poetry shell`                 | Activates the Poetry-managed virtual environment.             |
-
----
-
-## Additional Resources
-
-- **Poetry Documentation**: [https://python-poetry.org/docs/](https://python-poetry.org/docs/)
-- **GitHub Repository**: [https://github.com/python-poetry/poetry](https://github.com/python-poetry/poetry)
-
-For further help, please refer to the [Poetry documentation](https://python-poetry.org/docs/).

docs/adding_new_parser.md

@@ -0,0 +1,178 @@
+# Adding a New Dataset Parser
+
+This guide explains how to add a new dataset parser to the llmdataparser library. The library is designed to make it easy to add support for new datasets while maintaining consistent interfaces and functionality.
+
+## Step-by-Step Guide
+
+### 1. Create a New Parser Class
+
+Create a new file `your_dataset_parser.py` in the `llmdataparser` folder. Your parser should inherit from `HuggingFaceDatasetParser[T]` where T is your custom entry type.
+
+```python
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class YourDatasetParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for your dataset."""
+    # Add any additional fields specific to your dataset
+    custom_field: str
+
+    @classmethod
+    def create(cls, question: str, answer: str, raw_question: str,
+               raw_answer: str, task_name: str, custom_field: str) -> "YourDatasetParseEntry":
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            task_name=task_name,
+            custom_field=custom_field
+        )
+
+class YourDatasetParser(HuggingFaceDatasetParser[YourDatasetParseEntry]):
+    """Parser for your dataset."""
+
+    # Required class variables
+    _data_source = "huggingface/your-dataset"
+    _default_task = "default"
+    _task_names = ["task1", "task2", "task3"]
+```
+
+### 2. Implement Required Methods
+
+Your parser needs to implement these key methods:
+
+```python
+def process_entry(
+    self,
+    row: dict[str, Any],
+    task_name: str | None = None,
+    **kwargs: Any
+) -> YourDatasetParseEntry:
+    """Process a single dataset entry."""
+    # Extract data from the row
+    raw_question = row["question"]
+    raw_answer = row["answer"]
+    task = task_name or self._get_current_task(row)
+
+    question = f"Question: {raw_question}\nAnswer:"
+
+    return YourDatasetParseEntry.create(
+        question=question,
+        answer=raw_answer,
+        raw_question=raw_question,
+        raw_answer=raw_answer,
+        task_name=task,
+        custom_field=row["custom_field"]
+    )
+
+def get_dataset_description(self) -> DatasetDescription:
+    """Returns description of your dataset."""
+    return DatasetDescription.create(
+        name="Your Dataset Name",
+        purpose="Purpose of the dataset",
+        source="Dataset source/URL",
+        language="Dataset language",
+        format="Data format (e.g., multiple choice, free text)",
+        characteristics="Key characteristics of the dataset",
+        citation="Dataset citation if available"
+    )
+
+def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+    """Returns recommended evaluation metrics."""
+    return [
+        EvaluationMetric.create(
+            name="metric_name",
+            type="metric_type",
+            description="Metric description",
+            implementation="implementation_details",
+            primary=True
+        )
+    ]
+```
+
+### 3. Add Example Usage
+
+Add example usage at the bottom of your parser file:
+
+```python
+if __name__ == "__main__":
+    # Example usage
+    parser = YourDatasetParser()
+    parser.load()
+    parser.parse()
+
+    # Get parsed data
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Question: {example.raw_question}")
+        print(f"Answer: {example.answer}")
+```
+
+### 4. Create Tests
+
+Create a test file `tests/test_your_dataset_parser.py`:
+
+```python
+import pytest
+from llmdataparser.your_dataset_parser import YourDatasetParser, YourDatasetParseEntry
+
+def test_parser_initialization():
+    parser = YourDatasetParser()
+    assert parser._data_source == "huggingface/your-dataset"
+    assert parser._default_task == "default"
+    assert "task1" in parser._task_names
+
+def test_process_entry():
+    parser = YourDatasetParser()
+    sample_row = {
+        "question": "Sample question",
+        "answer": "Sample answer",
+        "custom_field": "Custom value"
+    }
+
+    entry = parser.process_entry(sample_row)
+    assert isinstance(entry, YourDatasetParseEntry)
+    assert entry.raw_question == "Sample question"
+    assert entry.custom_field == "Custom value"
+```
+
+## Best Practices
+
+1. **Type Safety**: Use type hints consistently and ensure your parser is properly typed.
+1. **Documentation**: Add clear docstrings and comments explaining your parser's functionality.
+1. **Error Handling**: Include appropriate error checking and validation.
+1. **Testing**: Write comprehensive tests covering different scenarios.
+
+## Examples
+
+Look at existing parsers for reference:
+
+- `mmlu_parser.py` for multiple-choice questions
+- `gsm8k_parser.py` for math word problems
+- `humaneval_parser.py` for code generation tasks
+
+## Common Patterns
+
+1. **Parse Entry Class**: Create a custom parse entry class if you need additional fields.
+1. **Task Names**: Define all available tasks in `_task_names`.
+1. **Process Entry**: Handle data extraction and formatting in `process_entry`.
+1. **Dataset Description**: Provide comprehensive dataset information.
+1. **Evaluation Metrics**: Define appropriate metrics for your dataset.
+
+## Testing Your Parser
+
+1. Run the example usage code to verify basic functionality
+1. Run pytest to execute your test cases
+1. Try different dataset splits and tasks
+1. Verify the parsed output format
+1. Check error handling with invalid inputs

llmdataparser/__init__.py

@@ -1,8 +1,22 @@
 # llmdataparser/__init__.py
-from typing import Type
+from typing import Any, Type
 
 from .base_parser import DatasetParser
-from .mmlu_parser import MMLUDatasetParser
+from .bbh_parser import BBHDatasetParser
+from .gsm8k_parser import GSM8KDatasetParser
+from .humaneval_parser import HumanEvalDatasetParser, HumanEvalDatasetPlusParser
+from .ifeval_parser import IFEvalDatasetParser
+from .math_parser import MATHDatasetParser
+from .mbpp_parser import MBPPDatasetParser
+from .mgsm_parser import MGSMDatasetParser
+from .mmlu_parser import (
+    BaseMMLUDatasetParser,
+    MMLUProDatasetParser,
+    MMLUReduxDatasetParser,
+    TMMLUPlusDatasetParser,
+)
+from .tmlu_parser import TMLUDatasetParser
+from .tw_legal_parser import TWLegalDatasetParser
 
 
 class ParserRegistry:
@@ -17,11 +31,13 @@ class ParserRegistry:
         cls._registry[name.lower()] = parser_class
 
     @classmethod
-    def get_parser(cls, name: str, **kwargs) -> Type[DatasetParser]:
+    def get_parser(cls, name: str, **kwargs: Any) -> DatasetParser[Any]:
+        """Get a parser instance by name."""
         parser_class = cls._registry.get(name.lower())
         if parser_class is None:
             raise ValueError(f"Parser '{name}' is not registered.")
-        return parser_class(**kwargs)
+        parser: DatasetParser[Any] = parser_class(**kwargs)
+        return parser
 
     @classmethod
     def list_parsers(cls) -> list[str]:
@@ -30,4 +46,17 @@ class ParserRegistry:
 
 
 # Register parsers
-ParserRegistry.register_parser("mmlu", MMLUDatasetParser)
+ParserRegistry.register_parser("mmlu", BaseMMLUDatasetParser)
+ParserRegistry.register_parser("mmlupro", MMLUProDatasetParser)
+ParserRegistry.register_parser("mmluredux", MMLUReduxDatasetParser)
+ParserRegistry.register_parser("tmmluplus", TMMLUPlusDatasetParser)
+ParserRegistry.register_parser("gsm8k", GSM8KDatasetParser)
+ParserRegistry.register_parser("math", MATHDatasetParser)
+ParserRegistry.register_parser("mgsm", MGSMDatasetParser)
+ParserRegistry.register_parser("humaneval", HumanEvalDatasetParser)
+ParserRegistry.register_parser("humanevalplus", HumanEvalDatasetPlusParser)
+ParserRegistry.register_parser("bbh", BBHDatasetParser)
+ParserRegistry.register_parser("mbpp", MBPPDatasetParser)
+ParserRegistry.register_parser("ifeval", IFEvalDatasetParser)
+ParserRegistry.register_parser("twlegal", TWLegalDatasetParser)
+ParserRegistry.register_parser("tmlu", TMLUDatasetParser)

llmdataparser/base_parser.py

@@ -1,7 +1,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from functools import lru_cache
-from typing import Any, Generic, TypeVar
+from typing import Any, ClassVar, Generic, TypeVar
 
 import datasets
 
@@ -9,17 +9,102 @@ import datasets
 T = TypeVar("T", bound="ParseEntry")
 
 
-@dataclass(frozen=True)
+# Add this after the DatasetCategory definition
+VALID_CATEGORIES = {
+    "Math",
+    "General Knowledge and Reasoning",
+    "Programming",
+    "MultiLingual",
+    "Taiwan",
+    "Advanced Reasoning",
+    "Legal",
+}
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
 class ParseEntry:
     """A simple base class for entries, customizable by each dataset parser."""
 
+    question: str
+    answer: str
+    raw_question: str
+    raw_answer: str
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class DatasetDescription:
+    """Standardized description of a dataset."""
+
+    name: str
+    purpose: str
+    source: str
+    language: str
+    format: str
+    category: list[str]
+    characteristics: str
+    citation: str | None = None
+    additional_info: dict[str, Any] | None = None
+
+    @classmethod
+    def create(
+        cls,
+        name: str,
+        purpose: str,
+        source: str,
+        language: str,
+        format: str,
+        category: list[str],
+        characteristics: str,
+        citation: str | None = None,
+        additional_info: dict[str, Any] | None = None,
+    ) -> "DatasetDescription":
+        # Validate that all categories are valid DatasetCategory values
+        for item in category:
+            assert (
+                item in VALID_CATEGORIES
+            ), f"Category '{item}' is not a valid category. Valid categories are: {VALID_CATEGORIES}"
+        return cls(
+            name=name,
+            purpose=purpose,
+            source=source,
+            language=language,
+            format=format,
+            category=category,
+            characteristics=characteristics,
+            citation=citation,
+            additional_info=additional_info,
+        )
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class EvaluationMetric:
+    """Description of an evaluation metric for a dataset."""
 
-class DatasetParser(ABC, Generic[T]):
+    name: str
+    type: str
+    description: str
+    implementation: str
+    primary: bool
+
+    @classmethod
+    def create(
+        cls, name: str, type: str, description: str, implementation: str, primary: bool
+    ) -> "EvaluationMetric":
+        return cls(
+            name=name,
+            type=type,
+            description=description,
+            implementation=implementation,
+            primary=primary,
+        )
+
+
+class DatasetParser(Generic[T], ABC):
     """
     Abstract base class defining the interface for all dataset parsers.
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._parsed_data: list[T] = []
 
     @abstractmethod
@@ -39,40 +124,189 @@ class DatasetParser(ABC, Generic[T]):
         return self._parsed_data
 
     @abstractmethod
-    def process_entry(self, row: dict[str, Any]) -> T:
-        pass
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> T:
+        """
+        Process a single entry from the dataset.
+
+        Args:
+            row: A dictionary representing a single entry from the dataset.
+            task_name: Optional task name for the entry.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            T: The processed entry, typically an instance of a subclass of ParseEntry.
+        """
+
+    @abstractmethod
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns a standardized description of the dataset."""
+
+    @abstractmethod
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for the dataset."""
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class HuggingFaceParseEntry(ParseEntry):
+    """ParseEntry with an additional task_name field."""
+
+    task_name: str
 
 
-# Base class for Hugging Face datasets
 class HuggingFaceDatasetParser(DatasetParser[T]):
     """
     Base class for parsers that use datasets from Hugging Face.
     """
 
-    _data_source: str  # Class variable for the dataset name
+    # _data_source is the name of the dataset, e.g. "lighteval/MATH"
+    _data_source: ClassVar[str]
+    # _task_names is the list of tasks in the dataset, e.g. ["algebra", "geometry", "statistics"]
+    _task_names: ClassVar[list[str]]
+    # _default_task is the default task to use if no task is specified, e.g. "algebra"
+    _default_task: ClassVar[str]
+    # _hidden_task_names is the list of task names that are hidden in the dataset, e.g. ["math", "physics", "chemistry"]
+    _hidden_task_names: ClassVar[list[str]] = []
 
-    def __init__(self):
-        self.raw_data = None
-        self.task_names = []
+    def __init__(self, **kwargs: Any) -> None:
+        """
+        Initialize a HuggingFaceDatasetParser.
+
+        Args:
+            **kwargs: Additional keyword arguments passed to the parent class.
+        """
         super().__init__()
+        # raw_data is the dataset loaded from HuggingFace
+        self.raw_data: dict[str, Any] | None = None
+        # split_names is the list of splits in the dataset, e.g. ["train", "test", "validation"]
+        self.split_names: list[str] = []
+        # _current_task is the task currently being processed, e.g. "algebra"
+        self._current_task: str = ""
+
+    def _get_current_task(self, data_entry: dict[str, Any] | None = None) -> str:
+        """
+        Get the currently loaded task name.
+
+        Args:
+            data_entry: Optional dictionary containing entry data that might include task information
+
+        Returns:
+            str: The task name from either the data entry (if available) or the currently set task
+        """
+        # If data_entry is provided and contains task information, use it
+        if data_entry is not None and hasattr(self, "_get_task_from_entry"):
+            try:
+                task = self._get_task_from_entry(data_entry)
+                if isinstance(task, str):  # Add type checking
+                    return task
+            except (KeyError, AttributeError):
+                pass
 
-    def get_task_names(self) -> list[str]:
-        return self.task_names
+        # Otherwise return the task set during load()
+        return self._current_task or self._default_task
+
+    @property
+    def task_names(self) -> list[str]:
+        """Get all available task names."""
+        return self._task_names
+
+    @property
+    def total_tasks(self) -> int:
+        """Get total number of available tasks."""
+        return len(self._task_names)
+
+    @property
+    def get_huggingface_link(self) -> str:
+        return "https://huggingface.co/datasets/" + self._data_source
 
     @staticmethod
     @lru_cache(maxsize=3)
     def load_dataset_cached(
-        data_source: str, config_name: str = "default", **kwargs: Any
-    ):
+        data_source: str,
+        task_name: str = "default",
+        trust_remote_code: bool = True,
+        **kwargs: Any,
+    ) -> datasets.Dataset:
         """
         Cached static method to load a dataset from Hugging Face.
         """
-        return datasets.load_dataset(data_source, config_name, **kwargs)
+        return datasets.load_dataset(
+            data_source, task_name, trust_remote_code=trust_remote_code, **kwargs
+        )
+
+    def parse(
+        self,
+        split_names: str | list[str] | None = None,
+        force: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        """
+        Parse the MATH dataset splits into structured entries.
+
+        Args:
+            split_names: Dataset splits to parse. Can be:
+                - None: Parse all available splits
+                - str: Parse a single split (e.g., "train")
+                - list[str]: Parse multiple splits (e.g., ["train", "test"])
+            force: If True, overwrites existing parsed data without confirmation.
+                If False and parsed data exists, prompts for confirmation.
+            **kwargs: Additional keyword arguments passed to process_entry
+
+        Raises:
+            ValueError: If no data is loaded or if a specified split name doesn't exist
+        """
+        if self.raw_data is None:
+            raise ValueError("No data loaded. Please load the dataset first.")
+
+        if self._parsed_data and not force:
+            response = input(
+                f"Found {len(self._parsed_data)} existing parsed entries. "
+                "Do you want to overwrite them? [y/N]: "
+            ).lower()
+            if response not in ("y", "yes"):
+                print("Parsing cancelled. Existing data preserved.")
+                return
+
+        self._parsed_data.clear()
+
+        # Dataset with splits
+        if split_names is None:
+            split_names = self.split_names
+        elif isinstance(split_names, str):
+            split_names = [split_names]
+
+        for split_name in split_names:
+            if split_name not in self.split_names:
+                raise ValueError(f"Split '{split_name}' not found in the dataset.")
+
+            dataset_split = self.raw_data[split_name]
+            total_entries = len(dataset_split)
+            print(f"Processing {split_name} split with {total_entries} entries...")
+
+            for index, entry in enumerate(dataset_split, start=1):
+                try:
+                    task_name = self._get_current_task(data_entry=entry)
+                    parsed_entry = self.process_entry(entry, task_name, **kwargs)
+                    self._parsed_data.append(parsed_entry)
+
+                    # Print progress every 100 entries
+                    if index % 100 == 0:
+                        print(
+                            f"Processed {index}/{total_entries} entries from '{split_name}'"
+                        )
+
+                except Exception as e:
+                    print(f"Error processing entry {index} in {split_name}: {str(e)}")
+                    continue
+
+            print(f"Completed parsing {index} entries from '{split_name}'")
+
+        print(f"Total parsed entries: {len(self._parsed_data)}")
 
     def load(
         self,
-        data_source: str | None = None,
-        config_name: str = "all",
+        task_name: str | None = None,
         trust_remote_code: bool = True,
         split: str | None = None,
         **kwargs: Any,
@@ -80,21 +314,41 @@ class HuggingFaceDatasetParser(DatasetParser[T]):
         """
         Load the dataset using the Hugging Face datasets library.
         """
-        # Use class-level data_source if not provided
-        data_source = data_source or self._data_source
-        if not data_source:
-            raise ValueError("The 'data_source' class variable must be defined.")
+        # Set the task name
+        self._current_task = task_name or self._default_task
 
         # Call the cached static method
-        self.raw_data = self.load_dataset_cached(
-            data_source,
-            config_name=config_name,
+        raw_data = self.load_dataset_cached(
+            self._data_source,
+            task_name=self._current_task,
             trust_remote_code=trust_remote_code,
             split=split,
             **kwargs,
         )
-        self.task_names = list(self.raw_data.keys())
+
+        # Handle split-specific loading
+        if split:
+            self.raw_data = {split: raw_data}
+            self.split_names = [split]
+        else:
+            self.raw_data = raw_data
+            self.split_names = list(raw_data.keys())
+
         print(
-            f"Loaded dataset with {len(self.task_names)} tasks: {', '.join(self.task_names)}."
+            f"Loaded dataset with {len(self.split_names)} groups: {', '.join(self.split_names)}."
+        )
+
+    def __repr__(self) -> str:
+        status = "loaded" if self.raw_data is not None else "not loaded"
+        parsed_count = len(self._parsed_data) if self._parsed_data else 0
+        return (
+            f"{self.__class__.__name__}("
+            f"data_source='{self._data_source}', "
+            f"task='{self._current_task}', "
+            f"status='{status}', "
+            f"parsed_entries={parsed_count}"
+            ")"
         )
-        # Additional common initialization can be added here
+
+    def __str__(self) -> str:
+        return self.__repr__()

llmdataparser/bbh_parser.py

@@ -0,0 +1,177 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar, List
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class BBHParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for BBH (Big Bench Hard), with fields specific to this dataset."""
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        task_name: str,
+    ) -> "BBHParseEntry":
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            task_name=task_name,
+        )
+
+
+class BBHDatasetParser(HuggingFaceDatasetParser[BBHParseEntry]):
+    """Parser for the Big Bench Hard dataset."""
+
+    _data_source: ClassVar[str] = "lukaemon/bbh"
+    _task_names: ClassVar[list[str]] = [
+        "boolean_expressions",
+        "causal_judgement",
+        "date_understanding",
+        "disambiguation_qa",
+        "dyck_languages",
+        "formal_fallacies",
+        "geometric_shapes",
+        "hyperbaton",
+        "logical_deduction_five_objects",
+        "logical_deduction_seven_objects",
+        "logical_deduction_three_objects",
+        "movie_recommendation",
+        "multistep_arithmetic_two",
+        "navigate",
+        "object_counting",
+        "penguins_in_a_table",
+        "reasoning_about_colored_objects",
+        "ruin_names",
+        "salient_translation_error_detection",
+        "snarks",
+        "sports_understanding",
+        "temporal_sequences",
+        "tracking_shuffled_objects_five_objects",
+        "tracking_shuffled_objects_seven_objects",
+        "tracking_shuffled_objects_three_objects",
+        "web_of_lies",
+        "word_sorting",
+    ]
+    _default_task: ClassVar[str] = "reasoning_about_colored_objects"
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> BBHParseEntry:
+        """Process a single BBH entry."""
+        raw_question = row["input"]
+        raw_answer = row["target"]
+
+        # Remove parentheses from the answer
+        clean_answer = raw_answer.strip("()")
+
+        question = str(raw_question)
+
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+
+        return BBHParseEntry.create(
+            question=question,
+            answer=clean_answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            task_name=task,
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns a description of the Big Bench Hard dataset."""
+        return DatasetDescription.create(
+            name="Big Bench Hard (BBH)",
+            purpose="A curated subset of 23 challenging BIG-Bench tasks where language models initially performed below average human-rater performance",
+            source="https://github.com/suzgunmirac/BIG-Bench-Hard",
+            language="English",
+            format="Multiple choice questions with single correct answers",
+            characteristics=(
+                "Tasks require complex multi-step reasoning and were selected based on "
+                "initial model performance below human baseline. Performance can be "
+                "significantly improved through chain-of-thought prompting. The dataset "
+                "includes 23 core tasks plus additional related tasks."
+            ),
+            category=["Advanced Reasoning"],
+            citation=(
+                "@article{suzgun2022challenging,\n"
+                "  title={Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them},\n"
+                '  author={Suzgun, Mirac and Scales, Nathan and Sch{"a}rli, Nathanael and Gehrmann, Sebastian and Tay, Yi and Chung, Hyung Won and Chowdhery, Aakanksha and Le, Quoc V and Chi, Ed H and Zhou, Denny and Wei, Jason},\n'
+                "  journal={arXiv preprint arXiv:2210.09261},\n"
+                "  year={2022}\n"
+                "}"
+            ),
+            additional_info={
+                "model_performance": (
+                    "With chain-of-thought prompting, PaLM surpassed human performance on "
+                    "10/23 tasks, while Codex surpassed human performance on 17/23 tasks"
+                ),
+                "size": "6.5k examples across 27 tasks (23 core + 4 related)",
+            },
+        )
+
+    def get_evaluation_metrics(self) -> List[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for BBH dataset."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Proportion of exactly correct answers (after stripping parentheses)",
+                implementation="evaluate.load('accuracy')",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="human_eval_delta",
+                type="comparison",
+                description="Difference between model accuracy and average human-rater performance baseline",
+                implementation="custom_human_baseline_comparison",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="per_task_accuracy",
+                type="classification",
+                description="Accuracy broken down by individual reasoning tasks",
+                implementation="custom_task_accuracy",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="exact_match",
+                type="string_match",
+                description="Strict exact match between predicted and target answers",
+                implementation="evaluate.load('exact_match')",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = BBHDatasetParser()
+
+    # Load the dataset with a specific task
+    parser.load(task_name="reasoning_about_colored_objects")
+
+    # Parse all splits
+    parser.parse()
+
+    # Get parsed data
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.question}")
+        print(f"Answer: {example.answer}")

llmdataparser/gsm8k_parser.py

@@ -0,0 +1,151 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class GSM8KParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for GSM8K, with fields specific to this dataset parser."""
+
+    solution: str
+    numerical_answer: int | float
+    task_name: str
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        solution: str,
+        numerical_answer: int | float,
+        task_name: str,
+    ) -> "GSM8KParseEntry":
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            solution=solution,
+            numerical_answer=numerical_answer,
+            task_name=task_name,
+        )
+
+
+class GSM8KDatasetParser(HuggingFaceDatasetParser[GSM8KParseEntry]):
+    """Parser for the GSM8K dataset."""
+
+    _data_source: ClassVar[str] = "openai/gsm8k"
+    _task_names: ClassVar[list[str]] = ["main", "socratic"]
+    _default_task: ClassVar[str] = "main"
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> GSM8KParseEntry:
+        """Process a single GSM8K entry."""
+        task = task_name or self._get_current_task(row)
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+
+        # Extract numerical answer (always after '####' in GSM8K)
+        numerical_str = raw_answer.split("####")[-1].strip().replace(",", "")
+        # Convert string to number
+        try:
+            numerical_answer = float(numerical_str)
+            if numerical_answer.is_integer():
+                numerical_answer = int(numerical_answer)
+        except ValueError:
+            raise ValueError(f"Could not convert '{numerical_str}' to number")
+
+        # Extract solution (everything before '####')
+        solution = raw_answer.split("####")[0].strip()
+
+        question = str(raw_question)
+
+        return GSM8KParseEntry.create(
+            question=question,
+            answer=str(numerical_answer),
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            solution=solution,
+            numerical_answer=numerical_answer,  # Now guaranteed to be int or float
+            task_name=task,  # Guarantee non-None
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the GSM8K dataset."""
+        return DatasetDescription.create(
+            name="Grade School Math 8K (GSM8K)",
+            purpose="Evaluate mathematical reasoning capabilities through word problems",
+            source="OpenAI",
+            language="English",
+            format="Word problems with step-by-step solutions and numerical answers",
+            category=["Math"],
+            characteristics=(
+                "Collection of 8.5K grade school math word problems that require "
+                "multi-step reasoning. Problems gradually increase in difficulty "
+                "and cover basic arithmetic, word problems, and elementary algebra"
+            ),
+            citation="""@article{cobbe2021gsm8k,
+    title={Training Verifiers to Solve Math Word Problems},
+    author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},
+    journal={arXiv preprint arXiv:2110.14168},
+    year={2021}
+    }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for GSM8K."""
+        return [
+            EvaluationMetric.create(
+                name="exact_match",
+                type="string",
+                description="Exact match comparison between predicted and correct numerical answers",
+                implementation="custom_exact_match",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="solution_validity",
+                type="text",
+                description="Assessment of whether the solution steps are mathematically valid and complete",
+                implementation="custom_solution_validator",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="step_accuracy",
+                type="numerical",
+                description="Accuracy of intermediate calculation steps (e.g., <<48/2=24>>)",
+                implementation="custom_step_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="step_count",
+                type="numerical",
+                description="Analysis of the number of reasoning steps in solutions",
+                implementation="custom_step_counter",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    from pprint import pprint
+
+    parser = GSM8KDatasetParser()
+    parser.load()
+    parser.parse()
+
+    parsed_data = parser.get_parsed_data
+    pprint(parsed_data[0].question)
+    pprint(parsed_data[0].answer)
+    pprint(parsed_data[0].raw_question)
+    pprint(parsed_data[0].raw_answer)
+    pprint(parsed_data[0].solution)
+    pprint(parsed_data[0].numerical_answer)

llmdataparser/humaneval_parser.py

@@ -0,0 +1,273 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class HumanEvalParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for HumanEval, with fields specific to this dataset parser."""
+
+    task_id: str
+    task_name: str
+    entry_point: str
+    test: str
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        task_id: str,
+        entry_point: str,
+        test: str,
+        task_name: str,
+    ) -> "HumanEvalParseEntry":
+        if not task_id:
+            raise ValueError("Task ID cannot be empty")
+        if not entry_point:
+            raise ValueError("Entry point cannot be empty")
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=answer,  # In HumanEval, the canonical solution is the raw answer
+            task_id=task_id,
+            entry_point=entry_point,
+            test=test,
+            task_name=task_name,
+        )
+
+
+class HumanEvalDatasetParser(HuggingFaceDatasetParser[HumanEvalParseEntry]):
+    """Parser for the HumanEval dataset."""
+
+    _data_source: ClassVar[str] = "openai/openai_humaneval"
+    _default_task: ClassVar[str] = "openai_humaneval"
+    _task_names: ClassVar[list[str]] = ["openai_humaneval"]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> HumanEvalParseEntry:
+        """Process a single HumanEval entry."""
+        raw_question = row["prompt"]
+        answer = row["canonical_solution"]
+        task_id = row["task_id"]
+        entry_point = row["entry_point"]
+        test = row["test"]
+
+        question = str(raw_question)
+
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+
+        return HumanEvalParseEntry.create(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            task_id=task_id,
+            entry_point=entry_point,
+            test=test,
+            task_name=task,  # Guarantee non-None
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the HumanEval dataset."""
+        return DatasetDescription.create(
+            name="HumanEval",
+            purpose="Evaluate code generation capabilities through Python programming tasks",
+            source="OpenAI",
+            language="Python",
+            format="Function signatures with docstrings and unit tests",
+            category=["Programming"],
+            characteristics=(
+                "Collection of 164 hand-written Python programming problems. Each problem "
+                "includes a function signature, docstring, example test cases, and hidden unit "
+                "tests. Problems test basic programming, algorithms, and data structure skills"
+            ),
+            citation="""@article{chen2021codex,
+    title={Evaluating Large Language Models Trained on Code},
+    author={Mark Chen and Jerry Tworek and Heewoo Jun and Qiming Yuan and Henrique Ponde de Oliveira Pinto and Jared Kaplan and Harri Edwards and Yuri Burda and Nicholas Joseph and Greg Brockman and Alex Ray and Raul Puri and Gretchen Krueger and Michael Petrov and Heidy Khlaaf and Girish Sastry and Pamela Mishkin and Brooke Chan and Scott Gray and Nick Ryder and Mikhail Pavlov and Alethea Power and Lukasz Kaiser and Mohammad Bavarian and Clemens Winter and Philippe Tillet and Felipe Petroski Such and Dave Cummings and Matthias Plappert and Fotios Chantzis and Elizabeth Barnes and Ariel Herbert-Voss and William Hebgen Guss and Alex Nichol and Alex Paino and Nikolas Tezak and Jie Tang and Igor Babuschkin and Suchir Balaji and Shantanu Jain and William Saunders and Christopher Hesse and Andrew N. Carr and Jan Leike and Josh Achiam and Vedant Misra and Evan Morikawa and Alec Radford and Matthew Knight and Miles Brundage and Mira Murati and Katie Mayer and Peter Welinder and Bob McGrew and Dario Amodei and Sam McCandlish and Ilya Sutskever and Wojciech Zaremba},
+    year={2021},
+    eprint={2107.03374},
+    archivePrefix={arXiv},
+    primaryClass={cs.LG}
+    }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for HumanEval."""
+        return [
+            EvaluationMetric.create(
+                name="pass@k",
+                type="code",
+                description="Probability that correct solution appears at least once in k samples",
+                implementation="custom_pass_at_k",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="test_success_rate",
+                type="code",
+                description="Percentage of test cases passed by the generated solution",
+                implementation="custom_test_executor",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="type_correctness",
+                type="code",
+                description="Verification of type hints and type safety in generated code",
+                implementation="custom_type_checker",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="code_style",
+                type="code",
+                description="Compliance with Python best practices and PEP 8 guidelines",
+                implementation="custom_style_checker",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="runtime_efficiency",
+                type="code",
+                description="Analysis of time and space complexity of the solution",
+                implementation="custom_complexity_analyzer",
+                primary=False,
+            ),
+        ]
+
+
+class HumanEvalDatasetPlusParser(HumanEvalDatasetParser):
+    """Parser for the enhanced HumanEval Plus dataset with 80x more comprehensive test coverage."""
+
+    _data_source: ClassVar[str] = "evalplus/humanevalplus"
+    _default_task: ClassVar[str] = "default"
+    _task_names: ClassVar[list[str]] = ["default"]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> HumanEvalParseEntry:
+        """Process a single HumanEval entry."""
+        raw_question = row["prompt"]
+        answer = row["canonical_solution"]
+        task_id = row["task_id"]
+        entry_point = row["entry_point"]
+        test = row["test"]
+
+        question = str(raw_question)
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+
+        return HumanEvalParseEntry.create(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            task_id=task_id,
+            entry_point=entry_point,
+            test=test,
+            task_name=task,  # task is guaranteed to be str from _get_current_task
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the HumanEval Plus dataset."""
+        return DatasetDescription.create(
+            name="HumanEval Plus",
+            purpose="Enhanced evaluation of code generation with 80x more test coverage",
+            source="EvalPlus",
+            language="Python",
+            format="Function signatures with docstrings and comprehensive test suites",
+            category=["Programming"],
+            characteristics=(
+                "Significantly enhanced version of HumanEval with 80x more test cases. "
+                "Includes extensive edge cases, boundary conditions, stress tests, and "
+                "error handling scenarios to rigorously evaluate code correctness and robustness. "
+                "Each problem has been augmented with comprehensive testing to catch subtle bugs "
+                "and ensure production-quality code generation."
+            ),
+            citation="""@inproceedings{evalplus,
+    title = {Is Your Code Generated by Chat{GPT} Really Correct? Rigorous Evaluation of Large Language Models for Code Generation},
+    author = {Liu, Jiawei and Xia, Chunqiu Steven and Wang, Yuyao and Zhang, Lingming},
+    booktitle = {Thirty-seventh Conference on Neural Information Processing Systems},
+    year = {2023},
+    url = {https://openreview.net/forum?id=1qvx610Cu7},
+    }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for HumanEval Plus."""
+        return [
+            EvaluationMetric.create(
+                name="pass@k",
+                type="code",
+                description="Probability that correct solution appears at least once in k samples",
+                implementation="custom_pass_at_k",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="test_coverage",
+                type="code",
+                description="Percentage of edge cases and stress tests passed by the solution",
+                implementation="custom_coverage_checker",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="error_handling",
+                type="code",
+                description="Assessment of solution's robustness in handling invalid inputs and edge cases",
+                implementation="custom_error_handler",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="performance_stress",
+                type="code",
+                description="Evaluation of solution performance under high load and stress conditions",
+                implementation="custom_stress_tester",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="code_quality",
+                type="code",
+                description="Analysis of code readability, maintainability and adherence to Python best practices",
+                implementation="custom_quality_checker",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = HumanEvalDatasetParser()
+
+    # Load the dataset
+    parser.load()
+
+    # Parse all splits
+    parser.parse()
+
+    # Get parsed data
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task ID: {example.task_id}")
+        print(f"Entry Point: {example.entry_point}")
+        print(f"Question:\n{example.question}")
+        print(f"Solution:\n{example.answer}")
+
+    parser = HumanEvalDatasetPlusParser()
+    parser.load()
+    parser.parse()
+    parsed_data = parser.get_parsed_data
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.raw_question}")
+        print(f"Correct Answer: {example.answer}")

llmdataparser/ifeval_parser.py

@@ -0,0 +1,164 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar, List
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class IFEvalParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for IFEval, with fields specific to this dataset parser."""
+
+    key: int
+    instruction_id_list: List[str]
+    kwargs: dict[str, Any]
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        key: int,
+        instruction_id_list: List[str],
+        kwargs: dict[str, Any],
+        task_name: str,
+    ) -> "IFEvalParseEntry":
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            key=key,
+            instruction_id_list=instruction_id_list,
+            kwargs=kwargs,
+            task_name=task_name,
+        )
+
+
+class IFEvalDatasetParser(HuggingFaceDatasetParser[IFEvalParseEntry]):
+    """Parser for the IFEval dataset."""
+
+    _data_source: ClassVar[str] = "google/IFEval"
+    _default_task: ClassVar[str] = "default"
+    _task_names: ClassVar[list[str]] = ["default"]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> IFEvalParseEntry:
+        """Process a single IFEval entry."""
+        # Extract fields from the row
+        key = row["key"]
+        raw_question = row["prompt"]  # The prompt is the raw question in this case
+        instruction_id_list = row["instruction_id_list"]
+        kwargs_data = row["kwargs"]
+
+        # For IFEval, we don't have explicit answers in the dataset
+        # We'll use empty strings as placeholders
+        answer = ""
+        raw_answer = ""
+
+        question = str(raw_question)
+
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+
+        return IFEvalParseEntry.create(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            key=key,
+            instruction_id_list=instruction_id_list,
+            kwargs=kwargs_data,
+            task_name=task,
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the IFEval dataset."""
+        return DatasetDescription.create(
+            name="IFEval",
+            purpose="Evaluate instruction following capabilities through verifiable instructions",
+            source="Google Research",
+            language="English (BCP-47 en)",
+            format="Verifiable instruction prompts with automated evaluation criteria",
+            category=["Programming"],
+            characteristics=(
+                "Collection of approximately 500 verifiable instructions designed to evaluate "
+                "language models' instruction-following capabilities. Instructions include "
+                "specific, measurable criteria like 'write in more than 400 words' or "
+                "'mention the keyword AI at least 3 times' that can be verified through "
+                "automated heuristics. Used as a core benchmark in the Open LLM Leaderboard "
+                "for evaluating chat or instruction fine-tuned language models."
+            ),
+            citation="""@misc{zhou2023instructionfollowingevaluationlargelanguage,
+    title={Instruction-Following Evaluation for Large Language Models},
+    author={Jeffrey Zhou and Tianjian Lu and Swaroop Mishra and Siddhartha Brahma and Sujoy Basu and Yi Luan and Denny Zhou and Le Hou},
+    year={2023},
+    eprint={2311.07911},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL},
+    url={https://arxiv.org/abs/2311.07911}
+    }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for IFEval."""
+        return [
+            EvaluationMetric.create(
+                name="format_compliance",
+                type="text",
+                description="Verifies if the output follows specified formatting rules (e.g., highlighting, bullet points, sections)",
+                implementation="custom_format_checker",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="length_constraints",
+                type="text",
+                description="Checks if the response meets word, sentence, or paragraph count requirements",
+                implementation="custom_length_validator",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="punctuation_rules",
+                type="text",
+                description="Validates adherence to punctuation constraints (e.g., no commas, specific endings)",
+                implementation="custom_punctuation_checker",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="keyword_usage",
+                type="text",
+                description="Verifies correct usage of required keywords or avoidance of forbidden words",
+                implementation="custom_keyword_validator",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="structural_requirements",
+                type="text",
+                description="Checks for specific structural elements like sections, paragraphs, or formatting patterns",
+                implementation="custom_structure_validator",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = IFEvalDatasetParser()
+    parser.load()
+    parser.parse()
+
+    parsed_data = parser.get_parsed_data
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Key: {example.key}")
+        print(f"Question: {example.question}")
+        print(f"Instruction IDs: {example.instruction_id_list}")
+        print(f"kwargs: {example.kwargs}")

llmdataparser/math_parser.py

@@ -0,0 +1,189 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MATHParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MATH dataset, with fields specific to this dataset parser."""
+
+    level: str
+    task_name: str
+    solution: str
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        level: str,
+        task_name: str,
+        solution: str,
+    ) -> "MATHParseEntry":
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            level=level,
+            task_name=task_name,
+            solution=solution,
+        )
+
+
+class MATHDatasetParser(HuggingFaceDatasetParser[MATHParseEntry]):
+    """Parser for the MATH dataset."""
+
+    _data_source: ClassVar[str] = "lighteval/MATH"
+    _task_names: ClassVar[list[str]] = [
+        "algebra",
+        "geometry",
+        "calculus",
+        "prealgebra",
+        "intermediate_algebra",
+        "number_theory",
+        "precalculus",
+        "all",
+    ]
+    _default_task: ClassVar[str] = "all"
+
+    _valid_levels: ClassVar[set[str]] = {
+        f"Level {i}" for i in range(1, 6)
+    }  # Levels 1-5 are valid
+
+    def _get_task_from_entry(self, data_entry: dict[str, Any]) -> str:
+        """Get the task name from the data entry or fall back to current task."""
+        entry_type: str = data_entry.get("type", "")
+        if entry_type and (entry_type in self._task_names):
+            return entry_type
+        return self._current_task or self._default_task
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MATHParseEntry:
+        """Process a single MATH dataset entry."""
+        task = task_name or self._get_current_task(row)
+
+        # Validate and normalize level
+        level = row.get("level")
+        if level not in self._valid_levels:
+            level = "Unknown"
+
+        return MATHParseEntry.create(
+            question=str(row["problem"]),
+            answer=row["solution"],
+            raw_question=row["problem"],
+            raw_answer=row["solution"],
+            level=level,
+            task_name=task,
+            solution=row["solution"],
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the MATH dataset."""
+        return DatasetDescription.create(
+            name="MATH",
+            purpose="Measure mathematical problem-solving capabilities in machine learning models",
+            source="Hendrycks et al., UC Berkeley (NeurIPS 2021)",
+            language="English",
+            format="Competition mathematics problems with step-by-step solutions",
+            category=["Math"],
+            characteristics=(
+                "Collection of 12,500 challenging competition mathematics problems designed to "
+                "evaluate mathematical reasoning. Problems include step-by-step solutions that "
+                "can be used to teach models to generate answer derivations and explanations. "
+                "Problems are categorized by subject area and difficulty level (1-5)."
+            ),
+            citation="""@article{hendrycksmath2021,
+    title={Measuring Mathematical Problem Solving With the MATH Dataset},
+    author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},
+    journal={NeurIPS},
+    year={2021}
+    }""",
+            additional_info={
+                "difficulty_levels": "1-5",
+                "topics": [
+                    "algebra",
+                    "geometry",
+                    "calculus",
+                    "prealgebra",
+                    "intermediate_algebra",
+                    "number_theory",
+                    "precalculus",
+                ],
+                "size": "12,500 problems",
+                "evaluation_note": "Exact match equivalence calculated using sympy library",
+                "homepage": "https://github.com/hendrycks/math",
+            },
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for MATH dataset."""
+        return [
+            EvaluationMetric.create(
+                name="symbolic_equivalence",
+                type="exact_match",
+                description="Verifies answer correctness using symbolic mathematics (e.g., sympy) to check mathematical equivalence.",
+                implementation="sympy_equivalence_checker",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="solution_presence",
+                type="text",
+                description="Ensures that a complete step-by-step solution is provided, demonstrating how the answer is derived.",
+                implementation="solution_completeness_checker",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="reasoning_validity",
+                type="text",
+                description="Evaluates the logical correctness and mathematical reasoning in the solution's derivation steps.",
+                implementation="reasoning_validator",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="mathematical_notation",
+                type="text",
+                description="Checks for the correct use of mathematical notation and symbolic representation to ensure clarity.",
+                implementation="notation_validator",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="solution_clarity",
+                type="text",
+                description="Assesses the clarity, readability, and coherence of the solution steps to enhance interpretability.",
+                implementation="clarity_scorer",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage of MATH parser
+    parser = MATHDatasetParser()
+
+    # Load the dataset
+    parser.load()
+
+    # Parse all splits
+    parser.parse()
+
+    # Get parsed data
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Level: {example.level}")
+        print(f"Question: {example.question}")
+        print(f"Solution: {example.solution}")

llmdataparser/mbpp_parser.py

@@ -0,0 +1,174 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MBPPParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MBPP, with fields specific to this dataset parser."""
+
+    task_id: int
+    test_list: list[str]
+    test_setup_code: str
+    challenge_test_list: list[str]
+    source_file: str
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        task_id: int,
+        test_list: list[str],
+        test_setup_code: str,
+        challenge_test_list: list[str],
+        task_name: str,
+        source_file: str,
+    ) -> "MBPPParseEntry":
+        if not isinstance(task_id, int):
+            raise ValueError("Task ID must be an integer")
+
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=answer,  # In MBPP, the code solution is the raw answer
+            task_id=task_id,
+            test_list=test_list,
+            test_setup_code=test_setup_code,
+            challenge_test_list=challenge_test_list,
+            task_name=task_name,
+            source_file=source_file,
+        )
+
+
+class MBPPDatasetParser(HuggingFaceDatasetParser[MBPPParseEntry]):
+    """Parser for the MBPP (Mostly Basic Python Programming) dataset."""
+
+    _data_source: ClassVar[str] = "google-research-datasets/mbpp"
+    _default_task: ClassVar[str] = "full"  # Can be 'full' or 'sanitized'
+    _task_names: ClassVar[list[str]] = ["full", "sanitized"]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MBPPParseEntry:
+        """Process a single MBPP entry."""
+        raw_question = row.get("text", row.get("prompt"))
+        answer = row["code"]
+        task_id = row["task_id"]
+        test_list = row["test_list"]
+        test_setup_code = row.get("test_setup_code", "")
+        challenge_test_list = row.get("challenge_test_list", [])
+
+        question = str(raw_question)
+
+        # Use task_name if provided, otherwise use default
+        task = task_name or self._get_current_task(row)
+        source_file = row.get("source_file", "")
+
+        return MBPPParseEntry.create(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            task_id=task_id,
+            test_list=test_list,
+            test_setup_code=test_setup_code,
+            challenge_test_list=challenge_test_list,
+            task_name=task,
+            source_file=source_file,
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns a description of the MBPP dataset."""
+        return DatasetDescription.create(
+            name="Mostly Basic Python Problems (MBPP)",
+            purpose="A benchmark for evaluating code generation capabilities using entry-level Python programming problems",
+            source="https://github.com/google-research/google-research/tree/master/mbpp",
+            language="English and Python",
+            category=["Programming"],
+            format="Task descriptions in English with corresponding Python solutions and automated test cases",
+            characteristics=(
+                "Contains approximately 1,000 crowd-sourced Python programming problems "
+                "designed for entry-level programmers. Problems cover programming fundamentals "
+                "and standard library functionality. Each problem includes a task description, "
+                "code solution, and 3 automated test cases. A subset of the data has been "
+                "hand-verified by the authors."
+            ),
+            citation=(
+                "@article{austin2021program,\n"
+                "  title={Program Synthesis with Large Language Models},\n"
+                "  author={Austin, Jacob and Odena, Augustus and Nye, Maxwell and Bosma, Maarten and Michalewski, Henryk and Dohan, David and Jiang, Ellen and Cai, Carrie and Terry, Michael and Le, Quoc and others},\n"
+                "  journal={arXiv preprint arXiv:2108.07732},\n"
+                "  year={2021}\n"
+                "}"
+            ),
+            additional_info={
+                "size": "~1,000 programming problems",
+                "splits": "Available in full or sanitized versions",
+                "test_coverage": "Each problem includes 3 automated test cases",
+                "verification": "Subset of data has been hand-verified by authors",
+            },
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for MBPP dataset."""
+        return [
+            EvaluationMetric.create(
+                name="pass@k",
+                type="code_evaluation",
+                description="Percentage of problems where at least one solution in k generations passes all test cases",
+                implementation="custom_pass_at_k",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="test_case_success_rate",
+                type="code_evaluation",
+                description="Percentage of test cases passed across all problems",
+                implementation="custom_test_success_rate",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="syntax_validity",
+                type="code_evaluation",
+                description="Verifies that generated code is syntactically valid Python",
+                implementation="custom_syntax_check",
+                primary=False,
+            ),
+            EvaluationMetric.create(
+                name="code_similarity",
+                type="similarity",
+                description="Similarity between generated code and reference solution",
+                implementation="evaluate.load('code_eval')",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = MBPPDatasetParser()
+
+    # Load the dataset
+    parser.load()
+
+    # Parse all splits
+    parser.parse()
+
+    # Get parsed data
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task ID: {example.task_id}")
+        print(f"Task: {example.raw_question}")
+        print(f"Solution:\n{example.answer}")
+        print(f"Test Cases:\n{example.test_list}")

llmdataparser/mgsm_parser.py

@@ -0,0 +1,192 @@
+from dataclasses import dataclass
+from typing import Any, ClassVar
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MGSMParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MGSM, with fields specific to this dataset parser."""
+
+    numerical_answer: int | float
+    equation_solution: str | None
+    language: str
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_answer: str,
+        numerical_answer: int | float,
+        equation_solution: str | None,
+        task_name: str,
+        language: str,
+    ) -> "MGSMParseEntry":
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            numerical_answer=numerical_answer,
+            equation_solution=equation_solution,
+            task_name=task_name,
+            language=language,
+        )
+
+
+class MGSMDatasetParser(HuggingFaceDatasetParser[MGSMParseEntry]):
+    """Parser for the MGSM (Multilingual Grade School Math) dataset."""
+
+    _data_source: ClassVar[str] = "juletxara/mgsm"
+    _default_task: ClassVar[str] = "en"
+    _task_names: ClassVar[list[str]] = [
+        "bn",
+        "de",
+        "en",
+        "es",
+        "fr",
+        "ja",
+        "ru",
+        "sw",
+        "te",
+        "th",
+        "zh",
+    ]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MGSMParseEntry:
+        """
+        Process a single MGSM entry.
+
+        Args:
+            row: Dictionary containing the MGSM entry fields
+            task_name: Language code for the current task
+
+        Returns:
+            MGSMParseEntry: Processed entry with question, answer, and metadata
+        """
+        task = task_name or self._get_current_task(row)
+        raw_question = row["question"]
+        raw_answer = row["answer"] if row["answer"] else ""
+        numerical_answer = row["answer_number"]
+        equation_solution = row["equation_solution"]
+
+        question = str(raw_question)
+
+        # Use numerical answer as string for the answer field if no detailed answer is provided
+        answer = raw_answer if raw_answer else str(numerical_answer)
+
+        return MGSMParseEntry.create(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            numerical_answer=numerical_answer,
+            equation_solution=equation_solution,
+            task_name=task,
+            language=task,
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns a description of the Multilingual Grade School Math dataset."""
+        return DatasetDescription.create(
+            name="Multilingual Grade School Math (MGSM)",
+            purpose="Evaluate multilingual chain-of-thought reasoning capabilities in mathematical problem solving",
+            source="https://huggingface.co/datasets/juletxara/mgsm",
+            language="Multilingual (11 languages)",
+            format="Word problems with numerical answers and solution steps",
+            category=["Math", "MultiLingual"],
+            characteristics=(
+                "Human-translated version of 250 GSM8K problems into 10 additional languages. "
+                "Each problem includes the original question from GSM8K, its translations, "
+                "numerical answer, and solution steps. The benchmark is designed to evaluate "
+                "language models' ability to perform mathematical reasoning across different languages."
+            ),
+            citation="""@misc{shi2022language,
+    title={Language Models are Multilingual Chain-of-Thought Reasoners},
+    author={Freda Shi and Mirac Suzgun and Markus Freitag and Xuezhi Wang and Suraj Srivats and Soroush Vosoughi and Hyung Won Chung and Yi Tay and Sebastian Ruder and Denny Zhou and Dipanjan Das and Jason Wei},
+    year={2022},
+    eprint={2210.03057},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+    }
+@article{cobbe2021gsm8k,
+    title={Training Verifiers to Solve Math Word Problems},
+    author={Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro and Hesse, Christopher and Schulman, John},
+    journal={arXiv preprint arXiv:2110.14168},
+    year={2021}
+}""",
+            additional_info={
+                "languages": [
+                    "Bengali",
+                    "German",
+                    "English",
+                    "Spanish",
+                    "French",
+                    "Japanese",
+                    "Russian",
+                    "Swahili",
+                    "Telugu",
+                    "Thai",
+                    "Chinese",
+                ],
+                "size": "250 problems translated into each language",
+                "base_dataset": "GSM8K (Grade School Math 8K)",
+            },
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for MGSM dataset."""
+        return [
+            EvaluationMetric.create(
+                name="exact_match",
+                type="string",
+                description="Exact match comparison between predicted and correct numerical answers",
+                implementation="custom_exact_match",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="solution_validity",
+                type="text",
+                description="Assessment of whether the solution steps are mathematically valid and complete",
+                implementation="custom_solution_validator",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="step_accuracy",
+                type="numerical",
+                description="Accuracy of intermediate calculation steps (e.g., <<48/2=24>>)",
+                implementation="custom_step_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="cross_lingual_consistency",
+                type="comparison",
+                description="Consistency of model performance across different language versions of the same problem",
+                implementation="custom_language_comparator",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    from pprint import pprint
+
+    parser = MGSMDatasetParser()
+    parser.load(task_name="en")  # Load French dataset
+    parser.parse()
+
+    parsed_data = parser.get_parsed_data
+    pprint(parsed_data[0].question)
+    pprint(parsed_data[0].answer)
+    pprint(parsed_data[0].raw_question)
+    pprint(parsed_data[0].numerical_answer)
+    pprint(parsed_data[0].language)

llmdataparser/mmlu_parser.py

@@ -1,81 +1,721 @@
 from dataclasses import dataclass
-from typing import Any
+from typing import Any, Final
 
-from llmdataparser.base_parser import HuggingFaceDatasetParser, ParseEntry
-from llmdataparser.prompts import MMLU_SYSTEM_PROMPT
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
 
+MMLU_VALID_ANSWERS: Final[set[str]] = {"A", "B", "C", "D"}
+MMLU_PRO_VALID_ANSWERS: Final[set[str]] = {
+    "A",
+    "B",
+    "C",
+    "D",
+    "E",
+    "F",
+    "G",
+    "H",
+    "I",
+    "J",
+}
+MMLU_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(MMLU_VALID_ANSWERS))
+MMLU_PRO_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(MMLU_PRO_VALID_ANSWERS))
 
-@dataclass(frozen=True)
-class MMLUParseEntry(ParseEntry):
-    """
-    Custom entry class for MMLU, with fields specific to this dataset parser.
-    """
 
-    prompt: str
-    answer_letter: str
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MMLUParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MMLU, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+    task_name: str
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+    ) -> "MMLUParseEntry":
+        if answer not in MMLU_VALID_ANSWERS:
+            raise ValueError(
+                f"Invalid answer_letter '{answer}'; must be one of {MMLU_VALID_ANSWER_STR}"
+            )
+        if not task_name:
+            raise ValueError("Task name cannot be empty")
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            raw_choices=raw_choices,
+            task_name=task_name,
+        )
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class MMLUProParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for MMLU, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+    task_name: str
 
     @classmethod
-    def create(cls, prompt: str, answer_letter: str) -> "MMLUParseEntry":
-        if answer_letter not in {"A", "B", "C", "D"}:
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+    ) -> "MMLUProParseEntry":
+        if answer not in MMLU_PRO_VALID_ANSWERS:
             raise ValueError(
-                f"Invalid answer_letter '{answer_letter}'; must be one of 'A', 'B', 'C', 'D'."
+                f"Invalid answer_letter '{answer}'; must be one of {MMLU_PRO_VALID_ANSWER_STR}"
             )
-        return cls(prompt=prompt, answer_letter=answer_letter)
+        if not task_name:
+            raise ValueError("Task name cannot be empty")
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=task_name,
+        )
 
 
 class MMLUDatasetParser(HuggingFaceDatasetParser[MMLUParseEntry]):
+    """Base class for MMLU dataset parsers with common functionality."""
+
+    def _get_task_from_entry(self, data_entry: dict[str, Any]) -> str:
+        """Get the task name from the data entry or default task name."""
+        task_name: str = data_entry.get("subject", "")
+        return task_name if task_name else (self._current_task or self._default_task)
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MMLUParseEntry:
+        """
+        Generate a question and expected answer from the given row.
+
+        Args:
+            row: A data point to be formatted.
+            task_name: Optional task name for the entry.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            MMLUParseEntry: The formatted entry object.
+        """
+        task = task_name or self._get_current_task(row)
+        # Ensure task is not None
+        final_task = task or self._default_task
+
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(row["choices"])
+        )
+        raw_question = row["question"]
+        raw_choices = row["choices"]
+        raw_answer = str(row["answer"])  # Ensure raw_answer is a string
+
+        question = f"Question: {raw_question}\n{choices}\nAnswer:"
+        answer_letter = chr(65 + int(raw_answer))  # Convert index to 'A', 'B', 'C', 'D'
+
+        return MMLUParseEntry.create(
+            question=question,
+            answer=answer_letter,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=final_task,
+        )
+
+
+class BaseMMLUDatasetParser(MMLUDatasetParser):
+    """Parser for the original MMLU dataset."""
+
     _data_source = "cais/mmlu"
+    _default_task = "all"
+    _task_names = [
+        "abstract_algebra",
+        "anatomy",
+        "astronomy",
+        "business_ethics",
+        "clinical_knowledge",
+        "college_biology",
+        "college_chemistry",
+        "college_computer_science",
+        "college_mathematics",
+        "college_medicine",
+        "college_physics",
+        "computer_security",
+        "conceptual_physics",
+        "econometrics",
+        "electrical_engineering",
+        "elementary_mathematics",
+        "formal_logic",
+        "global_facts",
+        "high_school_biology",
+        "high_school_chemistry",
+        "high_school_computer_science",
+        "high_school_european_history",
+        "high_school_geography",
+        "high_school_government_and_politics",
+        "high_school_macroeconomics",
+        "high_school_mathematics",
+        "high_school_microeconomics",
+        "high_school_physics",
+        "high_school_psychology",
+        "high_school_statistics",
+        "high_school_us_history",
+        "high_school_world_history",
+        "human_aging",
+        "human_sexuality",
+        "international_law",
+        "jurisprudence",
+        "logical_fallacies",
+        "machine_learning",
+        "management",
+        "marketing",
+        "medical_genetics",
+        "miscellaneous",
+        "moral_disputes",
+        "moral_scenarios",
+        "nutrition",
+        "philosophy",
+        "prehistory",
+        "professional_accounting",
+        "professional_law",
+        "professional_medicine",
+        "professional_psychology",
+        "public_relations",
+        "security_studies",
+        "sociology",
+        "us_foreign_policy",
+        "virology",
+        "world_religions",
+    ]
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns a description of the MMLU dataset."""
+        return DatasetDescription.create(
+            name="Massive Multitask Language Understanding (MMLU)",
+            purpose="Evaluate models' extensive world knowledge and problem-solving abilities across diverse branches of knowledge",
+            source="https://huggingface.co/datasets/cais/mmlu",
+            language="English",
+            category=["General Knowledge and Reasoning"],
+            format="Multiple choice questions with four options (A, B, C, D)",
+            characteristics=(
+                "Comprehensive evaluation benchmark spanning humanities, social sciences, hard sciences, "
+                "and other essential areas of knowledge. The test includes 57 subjects such as "
+                "elementary mathematics, US history, computer science, and law. Success on this test "
+                "requires both extensive world knowledge and strong problem-solving capabilities."
+            ),
+            citation="""@article{hendryckstest2021,
+    title={Measuring Massive Multitask Language Understanding},
+    author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt},
+    journal={Proceedings of the International Conference on Learning Representations (ICLR)},
+    year={2021}
+    }
+@article{hendrycks2021ethics,
+    title={Aligning AI With Shared Human Values},
+    author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},
+    journal={Proceedings of the International Conference on Learning Representations (ICLR)},
+    year={2021}
+    }""",
+            additional_info={
+                "subjects": "57 tasks/subjects",
+                "categories": [
+                    "Humanities",
+                    "Social Sciences",
+                    "Hard Sciences",
+                    "Other",
+                ],
+                "example_subjects": [
+                    "Elementary Mathematics",
+                    "US History",
+                    "Computer Science",
+                    "Law",
+                ],
+                "requirements": [
+                    "Extensive world knowledge",
+                    "Problem solving ability",
+                ],
+            },
+        )
 
-    def __init__(self, system_prompt: str = MMLU_SYSTEM_PROMPT):
-        super().__init__()  # Properly initialize the base class
-        self.parsed_data: list[MMLUParseEntry] = []
-        self.task_names: list[str] = []
-        self.subject_list: set[str] = set()
-        self.system_prompt: str = system_prompt
-        super().__init__()
-
-    def parse(self, split_names: str | list[str] | None = None, **kwargs: Any) -> None:
-        self.parsed_data.clear()
-        if self.raw_data is None:
-            raise ValueError("No data loaded. Please load the dataset first.")
-
-        if split_names is None:
-            split_names = self.task_names
-        elif isinstance(split_names, str):
-            split_names = [split_names]
-
-        for split_name in split_names:
-            if split_name not in self.task_names:
-                raise ValueError(f"Task '{split_name}' not found in the dataset.")
-
-            dataset_split = self.raw_data[split_name]
-            for index, entry in enumerate(dataset_split, start=1):
-                data_entry = self.process_entry(entry, **kwargs)
-                self._parsed_data.append(data_entry)
-                self.subject_list.add(entry.get("subject", "Unknown"))
-            print(f"Parsed {index} data points from task '{split_name}'.")
-
-        print(
-            f"Number of subjects: {len(self.subject_list)}. "
-            "For more details, please check the `self.subject_list` attribute."
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for MMLU dataset."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Proportion of correctly answered multiple-choice questions (exact match with A, B, C, D)",
+                implementation="evaluate.load('accuracy')",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="subject_accuracy",
+                type="classification",
+                description="Per-subject accuracy scores across all 57 tasks",
+                implementation="custom_subject_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="category_accuracy",
+                type="classification",
+                description="Accuracy grouped by major categories (Humanities, Social Sciences, Hard Sciences, Other)",
+                implementation="custom_category_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="task_correlation",
+                type="analysis",
+                description="Analysis of performance correlations between different subjects/tasks",
+                implementation="custom_task_correlation",
+                primary=False,
+            ),
+        ]
+
+
+class MMLUReduxDatasetParser(MMLUDatasetParser):
+    """Parser for the MMLU Redux dataset."""
+
+    _data_source = "edinburgh-dawg/mmlu-redux"
+    _default_task = "anatomy"
+    _task_names = [
+        "anatomy",
+        "astronomy",
+        "business_ethics",
+        "clinical_knowledge",
+        "college_chemistry",
+        "college_computer_science",
+        "college_mathematics",
+        "college_medicine",
+        "college_physics",
+        "conceptual_physics",
+        "econometrics",
+        "electrical_engineering",
+        "formal_logic",
+        "global_facts",
+        "high_school_chemistry",
+        "high_school_geography",
+        "high_school_macroeconomics",
+        "high_school_mathematics",
+        "high_school_physics",
+        "high_school_statistics",
+        "high_school_us_history",
+        "human_aging",
+        "logical_fallacies",
+        "machine_learning",
+        "miscellaneous",
+        "philosophy",
+        "professional_accounting",
+        "professional_law",
+        "public_relations",
+        "virology",
+    ]
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the MMLU Redux dataset."""
+        return DatasetDescription.create(
+            name="MMLU Redux",
+            purpose="Provide a manually re-annotated subset of MMLU with error analysis and corrections",
+            source="https://huggingface.co/datasets/edinburgh-dawg/mmlu-redux",
+            language="English",
+            format="Multiple choice questions with four options (A, B, C, D)",
+            category=["General Knowledge and Reasoning"],
+            characteristics=(
+                "A carefully curated subset of 3,000 questions across 30 MMLU subjects, "
+                "manually re-annotated to identify and classify various types of errors. "
+                "The dataset maintains the original questions but provides additional "
+                "error annotations and corrections based on expert review and verification "
+                "against credible sources."
+            ),
+            citation="""@misc{gema2024mmlu,
+    title={Are We Done with MMLU?},
+    author={Aryo Pradipta Gema and Joshua Ong Jun Leang and Giwon Hong and Alessio Devoto and Alberto Carlo Maria Mancino and Rohit Saxena and Xuanli He and Yu Zhao and Xiaotang Du and Mohammad Reza Ghasemi Madani and Claire Barale and Robert McHardy and Joshua Harris and Jean Kaddour and Emile van Krieken and Pasquale Minervini},
+    year={2024},
+    eprint={2406.04127},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL}
+    }""",
+            additional_info={
+                "size": "3,000 questions (100 per subject)",
+                "subjects": "30 MMLU subjects",
+                "license": "CC-BY-4.0",
+                "error_types": {
+                    "Question Assessment": [
+                        "Bad Question Clarity",
+                        "Bad Options Clarity",
+                    ],
+                    "Ground Truth Verification": [
+                        "No Correct Answer",
+                        "Multiple Correct Answers",
+                        "Wrong Ground Truth",
+                    ],
+                },
+                "verification_process": "Expert review with source verification",
+                "base_dataset": "cais/mmlu",
+            },
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for MMLU Redux dataset."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Proportion of correctly answered multiple-choice questions (exact match with A, B, C, D)",
+                implementation="evaluate.load('accuracy')",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="subject_accuracy",
+                type="classification",
+                description="Per-subject accuracy scores across 30 subjects (100 questions each)",
+                implementation="custom_subject_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="question_clarity",
+                type="analysis",
+                description="Analysis of performance on questions with different clarity issues",
+                implementation="custom_clarity_analysis",
+                primary=False,
+            ),
+        ]
+
+
+class TMMLUPlusDatasetParser(MMLUDatasetParser):
+    """Parser for the TMMLU+ dataset."""
+
+    _data_source = "ikala/tmmluplus"
+    _default_task = "taiwanese_hokkien"
+    _task_names = [
+        "engineering_math",
+        "dentistry",
+        "traditional_chinese_medicine_clinical_medicine",
+        "clinical_psychology",
+        "technical",
+        "culinary_skills",
+        "mechanical",
+        "logic_reasoning",
+        "real_estate",
+        "general_principles_of_law",
+        "finance_banking",
+        "anti_money_laundering",
+        "ttqav2",
+        "marketing_management",
+        "business_management",
+        "organic_chemistry",
+        "advance_chemistry",
+        "physics",
+        "secondary_physics",
+        "human_behavior",
+        "national_protection",
+        "jce_humanities",
+        "politic_science",
+        "agriculture",
+        "official_document_management",
+        "financial_analysis",
+        "pharmacy",
+        "educational_psychology",
+        "statistics_and_machine_learning",
+        "management_accounting",
+        "introduction_to_law",
+        "computer_science",
+        "veterinary_pathology",
+        "accounting",
+        "fire_science",
+        "optometry",
+        "insurance_studies",
+        "pharmacology",
+        "taxation",
+        "trust_practice",
+        "geography_of_taiwan",
+        "physical_education",
+        "auditing",
+        "administrative_law",
+        "education_(profession_level)",
+        "economics",
+        "veterinary_pharmacology",
+        "nautical_science",
+        "occupational_therapy_for_psychological_disorders",
+        "basic_medical_science",
+        "macroeconomics",
+        "trade",
+        "chinese_language_and_literature",
+        "tve_design",
+        "junior_science_exam",
+        "junior_math_exam",
+        "junior_chinese_exam",
+        "junior_social_studies",
+        "tve_mathematics",
+        "tve_chinese_language",
+        "tve_natural_sciences",
+        "junior_chemistry",
+        "music",
+        "education",
+        "three_principles_of_people",
+        "taiwanese_hokkien",
+    ]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MMLUParseEntry:
+        """Process a single TMMLU+ entry."""
+        # Extract choices in order
+        raw_choices = [row["A"], row["B"], row["C"], row["D"]]
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
+        )
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+
+        question = f"Question: {raw_question}\n{choices}\nAnswer:"
+        task = task_name or self._get_current_task(row)
+
+        return MMLUParseEntry.create(
+            question, raw_answer, raw_question, raw_choices, raw_answer, task
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the TMMLU+ dataset."""
+        return DatasetDescription.create(
+            name="Traditional Chinese Massive Multitask Language Understanding Plus (TMMLU+)",
+            purpose="Evaluate language models' understanding and reasoning capabilities in Traditional Chinese across diverse subjects",
+            source="https://huggingface.co/datasets/ikala/tmmluplus",
+            language="Traditional Chinese",
+            category=["General Knowledge and Reasoning", "Taiwan"],
+            format="Multiple choice questions with four options (A, B, C, D)",
+            characteristics=(
+                "A comprehensive evaluation benchmark featuring 66 subjects from elementary "
+                "to professional level. The dataset is six times larger than the original TMMLU "
+                "and provides more balanced subject coverage. Includes benchmark results from "
+                "both closed-source models and 20 open-weight Chinese language models with "
+                "parameters ranging from 1.8B to 72B."
+            ),
+            citation="""@article{ikala2024improved,
+    title={An Improved Traditional Chinese Evaluation Suite for Foundation Model},
+    author={Tam, Zhi-Rui and Pai, Ya-Ting and Lee, Yen-Wei and Cheng, Sega and Shuai, Hong-Han},
+    journal={arXiv preprint arXiv:2403.01858},
+    year={2024}
+    }""",
+            additional_info={
+                "subjects": "66 diverse subjects",
+                "difficulty_levels": ["Elementary", "Secondary", "Professional"],
+                "model_benchmarks": {
+                    "model_types": ["Closed-source models", "Open-weight Chinese LLMs"],
+                    "parameter_range": "1.8B - 72B",
+                },
+                "comparison": "6x larger than original TMMLU",
+                "script": "Traditional Chinese",
+            },
         )
 
-    def process_entry(self, row: dict[str, Any], **kwargs) -> MMLUParseEntry:
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for TMMLU+ dataset."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Overall percentage of correctly answered multiple-choice questions",
+                implementation="evaluate.load('accuracy')",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="subject_accuracy",
+                type="classification",
+                description="Per-subject accuracy scores across all 66 subjects",
+                implementation="custom_subject_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="difficulty_analysis",
+                type="classification",
+                description="Performance analysis across different difficulty levels (elementary to professional)",
+                implementation="custom_difficulty_analysis",
+                primary=False,
+            ),
+        ]
+
+
+class MMLUProDatasetParser(HuggingFaceDatasetParser[MMLUProParseEntry]):
+    """Parser for the MMLU Pro dataset."""
+
+    _data_source = "TIGER-Lab/MMLU-Pro"
+    _default_task = "default"
+    _task_names = ["default"]
+    _hidden_task_names = [
+        "math",
+        "physics",
+        "chemistry",
+        "law",
+        "engineering",
+        "other",
+        "economics",
+        "health",
+        "psychology",
+        "business",
+        "biology",
+        "philosophy",
+        "computer_science",
+        "history",
+    ]
+
+    def _get_task_from_entry(self, data_entry: dict[str, Any]) -> str:
+        """Get the task name from the data entry or default task name."""
+        if data_entry is not None:
+            task_name: str = data_entry.get("category", "")
+            if task_name:
+                return task_name
+        return self._current_task or self._default_task
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> MMLUProParseEntry:
         """
-        Generate a prompt and expected answer from the given row.
+        Generate a question and expected answer from the given row.
 
         Args:
-            row (dict[str, Any]): A data point to be formatted.
+            row (dict[str, Any]): A data point to be formatted with MMLU Pro specific structure
+                containing 'question', 'options', 'answer', and 'answer_index' keys.
 
         Returns:
             MMLUParseEntry: The formatted entry object.
         """
+        task = task_name or self._get_current_task(row)
+        # Ensure task is not None
+        final_task = task or self._default_task
+
+        # Extract choices in order
+        raw_choices = row["options"]
         choices = "\n".join(
-            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(row["choices"])
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
         )
-        prompt = (
-            f"{self.system_prompt}\nQuestion: {row['question']}\n{choices}\nAnswer:"
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+        answer_index = row["answer_index"]
+
+        question = f"Question: {raw_question}\n{choices}\nAnswer:"
+        answer_letter = chr(
+            65 + answer_index
+        )  # Convert index to 'A', 'B', 'C', 'D', etc.
+
+        return MMLUProParseEntry.create(
+            question, answer_letter, raw_question, raw_choices, raw_answer, final_task
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the MMLU Pro dataset."""
+        return DatasetDescription.create(
+            name="MMLU Pro",
+            purpose="Provide a more robust and challenging multi-task language understanding benchmark with enhanced reasoning requirements",
+            source="https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro",
+            language="English",
+            category=["General Knowledge and Reasoning", "Advanced Reasoning"],
+            format="Multiple choice questions with up to 10 options (expanded from original 4)",
+            characteristics=(
+                "A more challenging version of MMLU containing 12K complex questions across various "
+                "disciplines. Features increased number of options (up to 10), stronger focus on "
+                "reasoning over pure knowledge, and reduced sensitivity to prompt variations. "
+                "Questions are sourced from original MMLU, STEM websites, TheoremQA, and SciBench, "
+                "with expert review and GPT-4 assisted distractor generation."
+            ),
+            citation="""@article{wang2024mmlu,
+    title={Mmlu-pro: A more robust and challenging multi-task language understanding benchmark},
+    author={Wang, Yubo and Ma, Xueguang and Zhang, Ge and Ni, Yuansheng and Chandra, Abhranil and Guo, Shiguang and Ren, Weiming and Arulraj, Aaran and He, Xuan and Jiang, Ziyan and others},
+    journal={arXiv preprint arXiv:2406.01574},
+    year={2024}
+    }""",
+            additional_info={
+                "size": "12,000 complex questions",
+                "options": "Up to 10 choices per question",
+                "sources": [
+                    "Original MMLU (filtered)",
+                    "STEM Website",
+                    "TheoremQA",
+                    "SciBench",
+                ],
+                "enhanced_subjects": [
+                    "Biology",
+                    "Business",
+                    "Chemistry",
+                    "Computer Science",
+                    "Economics",
+                    "Engineering",
+                    "Math",
+                    "Physics",
+                    "Psychology",
+                ],
+                "construction_process": [
+                    "Initial MMLU filtering",
+                    "Question collection from multiple sources",
+                    "GPT-4 assisted option augmentation",
+                    "Expert review by 10+ experts",
+                ],
+                "prompt_sensitivity": "2% (reduced from 4-5% in MMLU)",
+                "reasoning_improvement": "20% higher CoT performance compared to PPL",
+            },
         )
-        answer_letter = chr(65 + row["answer"])  # Convert index to 'A', 'B', 'C', 'D'
 
-        return MMLUParseEntry.create(prompt, answer_letter)
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns the recommended evaluation metrics for MMLU Pro dataset."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Proportion of correctly answered multiple-choice questions (exact match)",
+                implementation="evaluate.load('accuracy')",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="subject_accuracy",
+                type="classification",
+                description="Per-subject accuracy scores with focus on enhanced subjects",
+                implementation="custom_subject_accuracy",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="reasoning_analysis",
+                type="analysis",
+                description="Comparison of Chain-of-Thought vs standard PPL performance",
+                implementation="custom_reasoning_analysis",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="prompt_robustness",
+                type="analysis",
+                description="Analysis of performance stability across different prompt variations",
+                implementation="custom_prompt_sensitivity",
+                primary=False,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage of MMLU Pro parser
+    parser = MMLUProDatasetParser()
+    parser.load()
+    parser.parse()
+
+    # Get parsed data with correct type
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.raw_question}")
+        print("Choices:")
+        for i, choice in enumerate(example.raw_choices):
+            print(f"{chr(65 + i)}. {choice}")
+        print(f"Correct Answer: {example.answer}")

llmdataparser/prompts.py

@@ -1,12 +1,69 @@
 import textwrap
 from typing import Final
 
+# Only for reference
+
 MMLU_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
     """\
-    You are an expert assistant for answering questions in a multiple-choice format. Each question has four answer options (A, B, C, D). Your task is to analyze each question and select the most accurate answer.
+    You are an expert answering multiple-choice questions. Select the single most accurate answer (A, B, C, or D) based on factual knowledge. Respond with the letter only.
+"""
+)
+
+MMLU_PRO_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    You are an expert answering multiple-choice questions. Select the single most accurate answer (A through J) based on factual knowledge. Respond with the letter only.
+"""
+)
+
+GSM8K_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Solve this math problem step by step:
+    1) Show your reasoning
+    2) End with "Therefore, the answer is [number]"
+"""
+)
+
+HUMANEVAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Implement the Python function following best practices. Include error handling, type hints, and comments for complex logic. Return only the implementation code.
+"""
+)
+
+MGSM_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Solve this math problem step by step in the specified language:
+    1) Show your reasoning
+    2) Use appropriate number formatting
+    3) End with "Therefore, the answer is [number]"
+"""
+)
+
+IFEVAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Follow the given requirements exactly. Provide only the requested output.
+"""
+)
+
+BBH_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Solve this reasoning problem step by step.
+"""
+)
+
+MBPP_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Write clean, efficient Python code that solves the given task. Include docstrings and handle edge cases. Return only the implementation code.
+"""
+)
 
-    Instructions:
-    1. Answer Selection: Review the question and choose the best option.
-    2. Response Format: Reply with only the letter (A, B, C, or D) of your chosen answer, without additional explanation.
+TW_LEGAL_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    As a Taiwan legal expert, select the most accurate answer (A, B, C, or D) based on Taiwan's laws. Respond with the letter only.
+"""
+)
+
+TMLU_SYSTEM_PROMPT: Final[str] = textwrap.dedent(
+    """\
+    Select the most accurate answer (A, B, C, or D) based on Taiwan's educational and professional knowledge. Respond with the letter only.
 """
 )

llmdataparser/tmlu_parser.py

@@ -0,0 +1,195 @@
+from dataclasses import dataclass
+from typing import Any, Final
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+TMLU_VALID_ANSWERS: Final[set[str]] = {"A", "B", "C", "D"}
+TMLU_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(TMLU_VALID_ANSWERS))
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class TMLUParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for TMLU, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+    explanation: str
+    metadata: dict[str, Any]
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+        explanation: str = "",
+        metadata: dict[str, Any] = {},
+    ) -> "TMLUParseEntry":
+        if answer not in TMLU_VALID_ANSWERS:
+            raise ValueError(
+                f"Invalid answer_letter '{answer}'; must be one of {TMLU_VALID_ANSWER_STR}"
+            )
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            raw_choices=raw_choices,
+            task_name=task_name,
+            explanation=explanation,
+            metadata=metadata,
+        )
+
+
+class TMLUDatasetParser(HuggingFaceDatasetParser[TMLUParseEntry]):
+    """Parser for the TMLU dataset."""
+
+    _data_source = "miulab/tmlu"
+    _default_task = "AST_chinese"
+    _task_names = [
+        "AST_chinese",
+        "AST_mathematics",
+        "AST_biology",
+        "AST_chemistry",
+        "AST_physics",
+        "AST_civics",
+        "AST_geography",
+        "AST_history",
+        "GSAT_chinese",
+        "GSAT_chemistry",
+        "GSAT_biology",
+        "GSAT_physics",
+        "GSAT_earth_science",
+        "GSAT_mathematics",
+        "GSAT_geography",
+        "GSAT_history",
+        "GSAT_civics",
+        "CAP_mathematics",
+        "CAP_biology",
+        "CAP_physics",
+        "CAP_chemistry",
+        "CAP_earth_science",
+        "CAP_civics",
+        "CAP_history",
+        "CAP_geography",
+        "CAP_chinese",
+        "driving_rule",
+        "basic_traditional_chinese_medicine",
+        "clinical_traditional_chinese_medicine",
+        "lawyer_qualification",
+        "nutritionist",
+        "tour_leader",
+        "tour_guide",
+        "taiwan_tourist_resources",
+        "clinical_psychologist",
+        "teacher_qualification",
+        "accountant",
+    ]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> TMLUParseEntry:
+        """Process a single TMLU entry."""
+        task = task_name or self._get_current_task(row)
+        # Extract choices in order
+        raw_choices = [row["A"], row["B"], row["C"], row["D"]]
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
+        )
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+        explanation = row.get("explanation", "")
+        metadata = row.get("metadata", {})
+
+        question = f"Question: {raw_question}\n{choices}\nAnswer:"
+
+        return TMLUParseEntry.create(
+            question=question,
+            answer=raw_answer,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=task,
+            explanation=explanation,
+            metadata=metadata,
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the TMLU dataset."""
+        return DatasetDescription.create(
+            name="Taiwan Multiple-choice Language Understanding (TMLU)",
+            language="Traditional Chinese",
+            purpose="Evaluate models on Taiwan-specific educational and professional knowledge",
+            source="Various Taiwan standardized tests and professional certifications",
+            category=["Taiwan", "General Knowledge and Reasoning"],
+            format="Multiple choice questions (A/B/C/D)",
+            characteristics=(
+                "Covers various subjects including Advanced Subjects Test (AST), "
+                "General Scholastic Ability Test (GSAT), College Admission Practice (CAP), "
+                "and professional certifications"
+            ),
+            citation="""@article{DBLP:journals/corr/abs-2403-20180,
+    author       = {Po-Heng Chen and Sijia Cheng and Wei-Lin Chen and Yen-Ting Lin and Yun-Nung Chen},
+    title        = {Measuring Taiwanese Mandarin Language Understanding},
+    journal      = {CoRR},
+    volume       = {abs/2403.20180},
+    year         = {2024},
+    url          = {https://doi.org/10.48550/arXiv.2403.20180},
+    doi          = {10.48550/ARXIV.2403.20180},
+    eprinttype   = {arXiv},
+    eprint       = {2403.20180},
+    timestamp    = {Wed, 10 Apr 2024 17:37:45 +0200},
+    biburl       = {https://dblp.org/rec/journals/corr/abs-2403-20180.bib},
+    bibsource    = {dblp computer science bibliography, https://dblp.org}
+    }""",
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for TMLU."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Overall percentage of correctly answered questions",
+                implementation="datasets.load_metric('accuracy')",
+                primary=True,
+            ),
+            EvaluationMetric.create(
+                name="per_subject_accuracy",
+                type="classification",
+                description="Accuracy broken down by subject areas (AST, GSAT, CAP, etc.)",
+                implementation="custom_subject_accuracy",
+                primary=True,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = TMLUDatasetParser()
+    parser.load()
+    parser.parse()
+
+    # Get parsed data with correct type
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Task: {example.task_name}")
+        print(f"Question: {example.question}")
+        print("Choices:")
+        for i, choice in enumerate(example.raw_choices):
+            print(f"{chr(65 + i)}. {choice}")
+        print(f"Correct Answer: {example.answer}")
+        if example.explanation:
+            print(f"Explanation: {example.explanation}")
+        print(f"Metadata: {example.metadata}")

llmdataparser/tw_legal_parser.py

@@ -0,0 +1,125 @@
+from dataclasses import dataclass
+from typing import Any, Final
+
+from llmdataparser.base_parser import (
+    DatasetDescription,
+    EvaluationMetric,
+    HuggingFaceDatasetParser,
+    HuggingFaceParseEntry,
+)
+
+TW_LEGAL_VALID_ANSWERS: Final[set[str]] = {"A", "B", "C", "D"}
+TW_LEGAL_VALID_ANSWER_STR: Final[str] = ", ".join(sorted(TW_LEGAL_VALID_ANSWERS))
+
+
+@dataclass(frozen=True, kw_only=True, slots=True)
+class TWLegalParseEntry(HuggingFaceParseEntry):
+    """Custom entry class for Taiwan Legal Benchmark, with fields specific to this dataset parser."""
+
+    raw_choices: list[str]
+
+    @classmethod
+    def create(
+        cls,
+        question: str,
+        answer: str,
+        raw_question: str,
+        raw_choices: list[str],
+        raw_answer: str,
+        task_name: str,
+    ) -> "TWLegalParseEntry":
+        if answer not in TW_LEGAL_VALID_ANSWERS:
+            raise ValueError(
+                f"Invalid answer_letter '{answer}'; must be one of {TW_LEGAL_VALID_ANSWER_STR}"
+            )
+        return cls(
+            question=question,
+            answer=answer,
+            raw_question=raw_question,
+            raw_answer=raw_answer,
+            raw_choices=raw_choices,
+            task_name=task_name,
+        )
+
+
+class TWLegalDatasetParser(HuggingFaceDatasetParser[TWLegalParseEntry]):
+    """Parser for the Taiwan Legal Benchmark dataset."""
+
+    _data_source = "lianghsun/tw-legal-benchmark-v1"
+    _default_task = "default"
+    _task_names = ["default"]
+
+    def process_entry(
+        self, row: dict[str, Any], task_name: str | None = None, **kwargs: Any
+    ) -> TWLegalParseEntry:
+        """Process a single Taiwan Legal Benchmark entry."""
+        # Extract choices in order
+        task = task_name or self._get_current_task(row)
+        raw_choices = [row["A"], row["B"], row["C"], row["D"]]
+        choices = "\n".join(
+            f"{chr(65 + i)}. {choice}" for i, choice in enumerate(raw_choices)
+        )
+        raw_question = row["question"]
+        raw_answer = row["answer"]
+
+        question = f"Question: {raw_question}\n{choices}\nAnswer:"
+
+        return TWLegalParseEntry.create(
+            question=question,
+            answer=raw_answer,
+            raw_question=raw_question,
+            raw_choices=raw_choices,
+            raw_answer=raw_answer,
+            task_name=task,
+        )
+
+    def get_dataset_description(self) -> DatasetDescription:
+        """Returns description of the Taiwan Legal Benchmark dataset."""
+        return DatasetDescription.create(
+            name="Taiwan Legal Benchmark",
+            language="Traditional Chinese",
+            purpose="Evaluate models on Taiwan-specific legal knowledge and understanding",
+            source="Taiwan Bar Examination questions",
+            category=["Taiwan", "General Knowledge and Reasoning", "Legal"],
+            format="Multiple choice questions (A/B/C/D)",
+            characteristics=(
+                "Contains questions from Taiwan's bar examination, testing understanding "
+                "of Taiwan's legal system, terminology, and concepts"
+            ),
+            citation="""
+                url={https://huggingface.co/datasets/lianghsun/tw-legal-benchmark-v1}
+            """,
+        )
+
+    def get_evaluation_metrics(self) -> list[EvaluationMetric]:
+        """Returns recommended evaluation metrics for Taiwan Legal Benchmark."""
+        return [
+            EvaluationMetric.create(
+                name="accuracy",
+                type="classification",
+                description="Overall percentage of correctly answered legal questions",
+                implementation="datasets.load_metric('accuracy')",
+                primary=True,
+            ),
+        ]
+
+
+if __name__ == "__main__":
+    # Example usage
+    parser = TWLegalDatasetParser()
+    parser.load()
+    parser.parse()
+
+    # Get parsed data with correct type
+    parsed_data = parser.get_parsed_data
+
+    # Print example entry
+    if parsed_data:
+        example = parsed_data[0]
+        print("\nExample parsed entry:")
+        print(f"Question: {example.question}")
+        print("Choices:")
+        for i, choice in enumerate(example.raw_choices):
+            print(f"{chr(65 + i)}. {choice}")
+        print(f"Correct Answer: {example.answer}")
+        print(f"Task Name: {example.task_name}")

mkdocs.yml

@@ -0,0 +1,9 @@
+site_name: LLMDataParser
+theme:
+  name: material
+
+nav:
+  - Home: index.md
+
+plugins:
+  - search

nginx.conf

@@ -0,0 +1,58 @@
+events {
+    worker_connections 1024;
+}
+
+http {
+    # Basic security settings
+    server_tokens off;  # Don't show nginx version
+    client_max_body_size 10M;  # Limit request size
+    client_body_timeout 12;
+    client_header_timeout 12;
+
+    upstream gradio_app {
+        server llmdataparser:7860;
+        keepalive 32;
+    }
+
+    server {
+        listen 80;
+        server_name localhost;
+
+        # Enhanced security headers
+        add_header X-Frame-Options "SAMEORIGIN" always;
+        add_header X-Content-Type-Options "nosniff" always;
+        add_header X-XSS-Protection "1; mode=block" always;
+        add_header Referrer-Policy "strict-origin-always" always;
+        add_header Content-Security-Policy "default-src 'self'; script-src 'self' 'unsafe-inline' 'unsafe-eval'; style-src 'self' 'unsafe-inline';" always;
+        add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
+
+        location / {
+            proxy_pass http://gradio_app;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            # WebSocket support
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            # Timeouts
+            proxy_connect_timeout 60s;
+            proxy_send_timeout 60s;
+            proxy_read_timeout 60s;
+
+            # Security
+            proxy_buffering on;
+            proxy_buffer_size 8k;
+            proxy_buffers 8 8k;
+        }
+
+        # Deny access to hidden files
+        location ~ /\. {
+            deny all;
+            return 404;
+        }
+    }
+}

notebooks/demo.ipynb

@@ -1,77 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pprint\n",
-    "import random"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from llmdataparser import ParserRegistry\n",
-    "ParserRegistry.list_parsers()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "mmlu_parser = ParserRegistry.get_parser('mmlu')\n",
-    "mmlu_parser.load()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "mmlu_parser.parse(split_names=['dev', 'test'])\n",
-    "parsed_data = mmlu_parser.get_parsed_data"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "index = random.randint(0, len(parsed_data))\n",
-    "print(f\"Question: \\n-------------------\\n {parsed_data[index].prompt}\")\n",
-    "print(\"-------------------\")\n",
-    "print(f\"Answer: \\n-------------------\\n{parsed_data[index].answer_letter}\")"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "llmdata",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.7"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}

pyproject.toml

@@ -1,6 +1,10 @@
+[build-system]
+requires = ["poetry-core>=1.5.0"]
+build-backend = "poetry.core.masonry.api"
+
 [tool.poetry]
 name = "llmdataparser"
-version = "0.1.0"
+version = "1.0.0"
 description = "A collection of parsers for LLM benchmark datasets like MMLU, MMLU-Pro, GSM8k, and more."
 authors = ["Jeff Yang <[email protected]>"]
 license = "MIT"
@@ -16,45 +20,66 @@ classifiers = [
     "Intended Audience :: Developers"
 ]
 
+packages = [
+    { include = "llmdataparser" }
+]
+
+[tool.poetry.scripts]
+start = "llmdataparser.app:main"
+
 [tool.poetry.dependencies]
-python = ">=3.11"
+python = ">=3.10"
 pandas = "^2.0.3"
-datasets = "^2.14.4"
+datasets = "^3.2.0"
+fsspec = "^2024.9.0"
 typing-extensions = "^4.8.0"
+ipywidgets = "^8.1.1"
+gradio = "^4.19.2"
+pyyaml = "^6.0.1"   # Add this for configuration handling
+tqdm = "^4.66.1"    # Add this for progress bars
+numpy = "^1.24.0"   # Add this for numerical operations
+mkdocs = "^1.5.0"
+mkdocs-material = "^9.5.0"  # Optional but recommended for better documentation
 
 [tool.poetry.group.dev.dependencies]
-pytest = "^7.4.0"
-black = { version = "^23.9.1", allow-prereleases = true }
-flake8 = "^6.1.0"
+pytest = "^7.0.0"
 isort = "^5.12.0"
 mypy = "^1.5.1"
 pre-commit = "^3.4.0"
 types-python-dateutil = "^2.8.19.14"
 ipykernel = "^6.7.0"
+coverage = "^7.4.1"
+pytest-cov = "^4.1.0"
+evaluate = "^0.4.0"
 
-[tool.black]
+[tool.ruff]
 line-length = 88
-target-version = ["py311"]
-exclude = """
-/(
-    \\.git
-  | \\.venv
-  | build
-  | dist
-)/
-"""
+
+[tool.ruff.lint]
+select = ["E", "F", "I"]
+ignore = ["E501"]
+
 
 [tool.isort]
 profile = "black"
+multi_line_output = 3
 line_length = 88
-known_first_party = ["llmdataparser"]
 
+[tool.mypy]
+python_version = "3.12"
+warn_return_any = true
+warn_unused_configs = true
+exclude = ["tests/.*"]
+ignore_missing_imports = true
+follow_imports = "silent"
 
-[tool.ruff]
-line-length = 88
-select = ["E", "F"]  # or specify checks explicitly without E501
-ignore = ["E501"]
 
-[build-system]
-requires = ["poetry-core>=1.5.0"]
-build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+python_files = ["test_*.py"]
+addopts = "-ra -q --cov=llmdataparser --cov-report=term-missing"
+
+[tool.bandit]
+exclude_dirs = ["tests"]
+skips = ["B101"]

tests/test_bbh_parser.py

@@ -0,0 +1,190 @@
+import pytest
+
+from llmdataparser.bbh_parser import BBHDatasetParser, BBHParseEntry
+
+
+@pytest.fixture
+def bbh_parser():
+    """Create a BBH parser instance for testing."""
+    return BBHDatasetParser()
+
+
+@pytest.fixture
+def loaded_bbh_parser(bbh_parser):
+    """Create and load a BBH parser instance for testing."""
+    bbh_parser.load(task_name="reasoning_about_colored_objects", split="test")
+    return bbh_parser
+
+
+@pytest.fixture
+def sample_row():
+    """Create a sample BBH data row for testing."""
+    return {
+        "input": "What color is the sky on a clear day?\nA) Blue\nB) Green\nC) Red\nD) Yellow",
+        "target": "(A)",
+    }
+
+
+def test_bbh_parse_entry_creation_valid():
+    """Test valid creation of BBHParseEntry."""
+    entry = BBHParseEntry.create(
+        question="Test question",
+        answer="A",
+        raw_question="Test question",
+        raw_answer="(A)",
+        task_name="reasoning_about_colored_objects",
+    )
+    assert isinstance(entry, BBHParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "A"
+    assert entry.raw_question == "Test question"
+    assert entry.raw_answer == "(A)"
+    assert entry.task_name == "reasoning_about_colored_objects"
+
+
+def test_bbh_parser_initialization(bbh_parser):
+    """Test BBH parser initialization."""
+    assert bbh_parser._data_source == "lukaemon/bbh"
+    assert bbh_parser._default_task == "reasoning_about_colored_objects"
+    assert "boolean_expressions" in bbh_parser._task_names
+    assert "word_sorting" in bbh_parser._task_names
+    assert (
+        bbh_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/lukaemon/bbh"
+    )
+
+
+def test_load_dataset(loaded_bbh_parser):
+    """Test loading the dataset."""
+    assert loaded_bbh_parser.raw_data is not None
+    assert loaded_bbh_parser.split_names == ["test"]
+    assert loaded_bbh_parser._current_task == "reasoning_about_colored_objects"
+
+
+@pytest.mark.integration
+def test_full_parse_workflow(loaded_bbh_parser):
+    """Test the complete workflow of loading and parsing data."""
+    # Parse the test split
+    loaded_bbh_parser.parse(split_names="test", force=True)
+    parsed_data = loaded_bbh_parser.get_parsed_data
+
+    # Basic checks
+    assert len(parsed_data) > 0
+
+    # Check first entry structure
+    first_entry = parsed_data[0]
+    assert isinstance(first_entry, BBHParseEntry)
+    assert first_entry.task_name == "reasoning_about_colored_objects"
+    assert first_entry.answer.strip("()").isalpha()  # Should be a single letter
+
+
+def test_process_entry(bbh_parser, sample_row):
+    """Test processing of a single BBH entry."""
+    entry = bbh_parser.process_entry(
+        sample_row, task_name="reasoning_about_colored_objects"
+    )
+
+    assert isinstance(entry, BBHParseEntry)
+    assert entry.answer == "A"  # Stripped from "(A)"
+    assert "What color is the sky" in entry.question
+    assert entry.raw_answer == "(A)"
+    assert entry.task_name == "reasoning_about_colored_objects"
+
+
+@pytest.mark.parametrize("split_name", ["invalid_split", "wrong_split"])
+def test_parse_with_invalid_split(bbh_parser, split_name):
+    """Test parsing with invalid split names."""
+    bbh_parser.raw_data = {"train": [], "test": []}  # Mock data
+
+    with pytest.raises(
+        ValueError, match=f"Split '{split_name}' not found in the dataset"
+    ):
+        bbh_parser.parse(split_name)
+
+
+def test_parse_without_loaded_data(bbh_parser):
+    """Test parsing without loading data first."""
+    with pytest.raises(
+        ValueError, match="No data loaded. Please load the dataset first"
+    ):
+        bbh_parser.parse()
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        {"input": "Test question", "target": "(A)"},
+        {"input": "Test question", "target": "(B)"},
+        {"input": "Test question", "target": "(C)"},
+    ],
+)
+def test_answer_stripping(bbh_parser, test_case):
+    """Test stripping of parentheses from answers."""
+    entry = bbh_parser.process_entry(
+        test_case, task_name="reasoning_about_colored_objects"
+    )
+    assert entry.answer == test_case["target"].strip("()")
+    assert entry.raw_answer == test_case["target"]
+
+
+def test_parser_properties(bbh_parser):
+    """Test parser property getters."""
+    assert len(bbh_parser.task_names) > 0
+    assert bbh_parser.total_tasks == len(bbh_parser._task_names)
+    assert all(isinstance(task, str) for task in bbh_parser.task_names)
+
+
+def test_parser_string_representation(loaded_bbh_parser):
+    """Test string representation of parser."""
+    repr_str = str(loaded_bbh_parser)
+    assert "BBHDatasetParser" in repr_str
+    assert "lukaemon/bbh" in repr_str
+    assert "reasoning_about_colored_objects" in repr_str
+    assert "loaded" in repr_str
+
+
+@pytest.mark.integration
+@pytest.mark.parametrize(
+    "task_name", ["boolean_expressions", "causal_judgement", "date_understanding"]
+)
+def test_different_tasks_parsing(bbh_parser, task_name):
+    """Test parsing different tasks of the dataset."""
+    bbh_parser.load(task_name=task_name, split="test")
+    bbh_parser.parse(split_names="test", force=True)
+    parsed_data = bbh_parser.get_parsed_data
+
+    assert len(parsed_data) > 0
+    assert all(entry.task_name == task_name for entry in parsed_data)
+    assert all(isinstance(entry.answer, str) for entry in parsed_data)
+
+
+def test_get_dataset_description(bbh_parser):
+    """Test dataset description generation."""
+    description = bbh_parser.get_dataset_description()
+
+    assert description.name == "Big Bench Hard (BBH)"
+    assert description.language == "English"
+    assert description.format == "Multiple choice questions with single correct answers"
+    assert "suzgun2022challenging" in description.citation
+
+
+def test_get_evaluation_metrics(bbh_parser):
+    """Test evaluation metrics generation."""
+    metrics = bbh_parser.get_evaluation_metrics()
+
+    assert len(metrics) == 4  # Check total number of metrics
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 2
+    assert any(m.name == "accuracy" for m in primary_metrics)
+    assert any(m.name == "human_eval_delta" for m in primary_metrics)
+
+    # Check specific metric properties
+    accuracy_metric = next(m for m in metrics if m.name == "accuracy")
+    assert accuracy_metric.type == "classification"
+    assert "evaluate.load('accuracy')" in accuracy_metric.implementation
+
+    # Check non-primary metrics
+    assert any(m.name == "per_task_accuracy" and not m.primary for m in metrics)
+    assert any(m.name == "exact_match" and not m.primary for m in metrics)

tests/test_gsm8k_parser.py

@@ -0,0 +1,207 @@
+import pytest
+
+from llmdataparser.gsm8k_parser import GSM8KDatasetParser, GSM8KParseEntry
+
+
+@pytest.fixture
+def gsm8k_parser():
+    """Create a GSM8K parser instance for testing."""
+    return GSM8KDatasetParser()
+
+
+@pytest.fixture
+def loaded_gsm8k_parser(gsm8k_parser):
+    """Create and load a GSM8K parser instance for testing."""
+    gsm8k_parser.load(
+        task_name="main", split="test"
+    )  # Using test split as it's smaller
+    return gsm8k_parser
+
+
+@pytest.fixture
+def sample_row():
+    """Create a sample GSM8K data row for testing."""
+    return {
+        "question": "Janet has 3 apples. She buys 2 more. How many apples does she have now?",
+        "answer": "Let's solve this step by step:\n1) Initially, Janet has 3 apples\n2) She buys 2 more apples\n3) Total apples = 3 + 2\n#### 5",
+    }
+
+
+def test_gsm8k_parse_entry_creation_valid():
+    """Test valid creation of GSM8KParseEntry."""
+    entry = GSM8KParseEntry.create(
+        question="Test question",
+        answer="5",
+        raw_question="Test question",
+        raw_answer="Solution steps #### 5",
+        solution="Solution steps",
+        task_name="main",
+        numerical_answer=5,
+    )
+    assert isinstance(entry, GSM8KParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "5"
+    assert entry.solution == "Solution steps"
+    assert entry.numerical_answer == 5
+    assert entry.task_name == "main"
+
+
+def test_gsm8k_parser_initialization(gsm8k_parser):
+    """Test GSM8K parser initialization."""
+    assert gsm8k_parser._data_source == "openai/gsm8k"
+    assert gsm8k_parser._default_task == "main"
+    assert gsm8k_parser._task_names == ["main", "socratic"]
+    assert (
+        gsm8k_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/openai/gsm8k"
+    )
+
+
+def test_load_dataset(loaded_gsm8k_parser):
+    """Test loading the dataset."""
+    assert loaded_gsm8k_parser.raw_data is not None
+    assert loaded_gsm8k_parser.split_names == [
+        "test"
+    ]  # Since we specifically loaded the test split
+    assert loaded_gsm8k_parser._current_task == "main"
+
+
+@pytest.mark.integration
+def test_full_parse_workflow(loaded_gsm8k_parser):
+    """Test the complete workflow of loading and parsing data."""
+    # Parse the test split
+    loaded_gsm8k_parser.parse(split_names="test", force=True)
+    parsed_data = loaded_gsm8k_parser.get_parsed_data
+
+    # Basic checks
+    assert len(parsed_data) > 0
+
+    # Check first entry structure
+    first_entry = parsed_data[0]
+    assert isinstance(first_entry, GSM8KParseEntry)
+    assert first_entry.task_name == "main"
+    assert isinstance(first_entry.numerical_answer, (str, int, float))
+    assert "####" in first_entry.raw_answer
+    assert first_entry.solution
+
+
+def test_process_entry(gsm8k_parser, sample_row):
+    """Test processing of a single GSM8K entry."""
+    entry = gsm8k_parser.process_entry(sample_row, task_name="main")
+
+    assert isinstance(entry, GSM8KParseEntry)
+    assert entry.numerical_answer == 5
+    assert "Janet has 3 apples" in entry.raw_question
+    assert "#### 5" in entry.raw_answer
+    assert "Let's solve this step by step:" in entry.solution
+    assert entry.task_name == "main"
+
+
+@pytest.mark.parametrize("split_name", ["invalid_split", "wrong_split"])
+def test_parse_with_invalid_split(gsm8k_parser, split_name):
+    """Test parsing with invalid split names."""
+    gsm8k_parser.raw_data = {"train": [], "test": []}  # Mock data
+
+    with pytest.raises(
+        ValueError, match=f"Split '{split_name}' not found in the dataset"
+    ):
+        gsm8k_parser.parse(split_name)
+
+
+def test_parse_without_loaded_data(gsm8k_parser):
+    """Test parsing without loading data first."""
+    with pytest.raises(
+        ValueError, match="No data loaded. Please load the dataset first"
+    ):
+        gsm8k_parser.parse()
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        {"question": "Test question", "answer": "Some solution steps #### 42"},
+        {
+            "question": "Test question",
+            "answer": "Complex solution\nWith multiple lines\n#### 123.45",
+        },
+        {"question": "Test question", "answer": "No steps #### 0"},
+    ],
+)
+def test_numerical_answer_extraction(gsm8k_parser, test_case):
+    """Test extraction of numerical answers from different formats."""
+    entry = gsm8k_parser.process_entry(test_case, task_name="main")
+    assert str(entry.numerical_answer) == test_case["answer"].split("####")[
+        -1
+    ].strip().replace(",", "")
+
+
+def test_solution_extraction(gsm8k_parser):
+    """Test extraction of solution steps."""
+    row = {
+        "question": "Test question",
+        "answer": "Step 1: Do this\nStep 2: Do that\n#### 42",
+    }
+
+    entry = gsm8k_parser.process_entry(row, task_name="main")
+    assert entry.solution == "Step 1: Do this\nStep 2: Do that"
+    assert entry.task_name == "main"
+    assert "####" not in entry.solution
+
+
+def test_parser_properties(gsm8k_parser):
+    """Test parser property getters."""
+    assert gsm8k_parser.task_names == ["main", "socratic"]
+    assert gsm8k_parser.total_tasks == 2
+
+
+def test_parser_string_representation(loaded_gsm8k_parser):
+    """Test string representation of parser."""
+    repr_str = str(loaded_gsm8k_parser)
+    assert "GSM8KDatasetParser" in repr_str
+    assert "openai/gsm8k" in repr_str
+    assert "main" in repr_str
+    assert "loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_splits_parsing(gsm8k_parser):
+    """Test parsing different splits of the dataset."""
+    # Load and parse test split
+    gsm8k_parser.load(task_name="main", split="test")
+    gsm8k_parser.parse(split_names="test", force=True)
+    test_count = len(gsm8k_parser.get_parsed_data)
+
+    # Load and parse train split
+    gsm8k_parser.load(task_name="main", split="train")
+    gsm8k_parser.parse(split_names="train", force=True)
+    train_count = len(gsm8k_parser.get_parsed_data)
+
+    assert test_count > 0
+    assert train_count > 0
+    assert train_count != test_count
+
+
+def test_get_dataset_description(gsm8k_parser):
+    """Test dataset description generation."""
+    description = gsm8k_parser.get_dataset_description()
+
+    assert description.name == "Grade School Math 8K (GSM8K)"
+    assert description.source == "OpenAI"
+    assert description.language == "English"
+    assert "Cobbe" in description.citation
+
+
+def test_get_evaluation_metrics(gsm8k_parser):
+    """Test evaluation metrics specification."""
+    metrics = gsm8k_parser.get_evaluation_metrics()
+
+    # Check we have all expected metrics
+    metric_names = {metric.name for metric in metrics}
+    expected_names = {"exact_match", "solution_validity", "step_accuracy", "step_count"}
+    assert metric_names == expected_names
+
+    # Check exact_match metric details
+    exact_match = next(m for m in metrics if m.name == "exact_match")
+    assert exact_match.type == "string"
+    assert exact_match.primary is True
+    assert "exact match" in exact_match.description.lower()

tests/test_humaneval_parser.py

@@ -0,0 +1,198 @@
+import pytest
+
+from llmdataparser.humaneval_parser import (
+    HumanEvalDatasetParser,
+    HumanEvalDatasetPlusParser,
+    HumanEvalParseEntry,
+)
+
+
+@pytest.fixture
+def sample_entry():
+    return {
+        "prompt": 'def add(a, b):\n    """Add two numbers."""\n',
+        "canonical_solution": "def add(a, b):\n    return a + b\n",
+        "task_id": "HumanEval/0",
+        "entry_point": "add",
+        "test": "def test_add(): assert add(2, 3) == 5",
+    }
+
+
+@pytest.fixture
+def parser():
+    return HumanEvalDatasetParser()
+
+
+@pytest.fixture
+def plus_parser():
+    return HumanEvalDatasetPlusParser()
+
+
+@pytest.fixture
+def plus_sample_entry():
+    return {
+        "prompt": 'def add(a, b):\n    """Add two numbers."""\n',
+        "canonical_solution": "def add(a, b):\n    return a + b\n",
+        "task_id": "HumanEval/0",
+        "entry_point": "add",
+        "test": "def test_add(): assert add(2, 3) == 5",
+    }
+
+
+def test_humaneval_parse_entry_creation():
+    """Test creation of HumanEvalParseEntry"""
+    entry = HumanEvalParseEntry.create(
+        question="test question",
+        answer="test answer",
+        raw_question="raw question",
+        task_id="HumanEval/1",
+        entry_point="test_func",
+        test="test case",
+        task_name="openai_humaneval",
+    )
+
+    assert entry.question == "test question"
+    assert entry.answer == "test answer"
+    assert entry.raw_question == "raw question"
+    assert entry.raw_answer == "test answer"  # Should match answer
+    assert entry.task_id == "HumanEval/1"
+    assert entry.entry_point == "test_func"
+    assert entry.test == "test case"
+    assert entry.task_name == "openai_humaneval"
+
+
+def test_humaneval_parse_entry_validation():
+    """Test validation of required fields"""
+    with pytest.raises(ValueError, match="Task ID cannot be empty"):
+        HumanEvalParseEntry.create(
+            question="test",
+            answer="test",
+            raw_question="test",
+            task_id="",  # Empty task_id should raise error
+            entry_point="test",
+            test="test",
+            task_name="test",
+        )
+
+    with pytest.raises(ValueError, match="Entry point cannot be empty"):
+        HumanEvalParseEntry.create(
+            question="test",
+            answer="test",
+            raw_question="test",
+            task_id="test",
+            entry_point="",  # Empty entry_point should raise error
+            test="test",
+            task_name="test",
+        )
+
+
+def test_process_entry(parser, sample_entry):
+    """Test processing of a single entry"""
+    result = parser.process_entry(sample_entry, task_name="openai_humaneval")
+
+    assert isinstance(result, HumanEvalParseEntry)
+    assert result.task_id == "HumanEval/0"
+    assert result.entry_point == "add"
+
+    assert result.answer == sample_entry["canonical_solution"]
+    assert result.test == sample_entry["test"]
+    assert result.task_name == "openai_humaneval"
+
+
+def test_parser_initialization(parser):
+    """Test parser initialization and properties"""
+    assert parser._data_source == "openai/openai_humaneval"
+    assert parser._default_task == "openai_humaneval"
+    assert parser._task_names == ["openai_humaneval"]
+    assert (
+        parser.get_huggingface_link
+        == "https://huggingface.co/datasets/openai/openai_humaneval"
+    )
+
+
+@pytest.mark.integration
+def test_parser_load_and_parse(parser):
+    """Integration test for loading and parsing data"""
+    parser.load()
+    parser.parse()
+    parsed_data = parser.get_parsed_data
+
+    assert len(parsed_data) > 0
+    assert all(isinstance(entry, HumanEvalParseEntry) for entry in parsed_data)
+
+
+def test_get_current_task(parser, sample_entry):
+    """Test _get_current_task method"""
+    task = parser._get_current_task(sample_entry)
+    assert task == parser._default_task
+
+
+def test_plus_parser_initialization(plus_parser):
+    """Test HumanEvalDatasetPlusParser initialization and properties"""
+    assert plus_parser._data_source == "evalplus/humanevalplus"
+    assert plus_parser._default_task == "default"
+    assert plus_parser._task_names == ["default"]
+    assert (
+        plus_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/evalplus/humanevalplus"
+    )
+
+
+def test_plus_process_entry(plus_parser, plus_sample_entry):
+    """Test processing of a single entry in HumanEvalDatasetPlusParser"""
+    result = plus_parser.process_entry(plus_sample_entry, task_name="default")
+
+    assert isinstance(result, HumanEvalParseEntry)
+    assert result.task_id == "HumanEval/0"
+    assert result.entry_point == "add"
+
+    assert result.answer == plus_sample_entry["canonical_solution"]
+    assert result.test == plus_sample_entry["test"]
+    assert result.task_name == "default"
+
+
+@pytest.mark.integration
+def test_plus_parser_load_and_parse(plus_parser):
+    """Integration test for loading and parsing data with HumanEvalDatasetPlusParser"""
+    plus_parser.load()
+    plus_parser.parse()
+    parsed_data = plus_parser.get_parsed_data
+
+    assert len(parsed_data) > 0
+    assert all(isinstance(entry, HumanEvalParseEntry) for entry in parsed_data)
+
+
+def test_plus_get_current_task(plus_parser, plus_sample_entry):
+    """Test _get_current_task method for HumanEvalDatasetPlusParser"""
+    task = plus_parser._get_current_task(plus_sample_entry)
+    assert task == plus_parser._default_task
+
+
+def test_get_dataset_description(parser, plus_parser):
+    """Test dataset description generation for both parsers."""
+    # Test original HumanEval description
+    description = parser.get_dataset_description()
+    assert description.name == "HumanEval"
+    assert "code generation" in description.purpose
+    assert description.language == "Python"
+    assert "chen2021codex" in description.citation
+
+    # Test HumanEval Plus description
+    plus_description = plus_parser.get_dataset_description()
+    assert plus_description.name == "HumanEval Plus"
+    assert "80x more test coverage" in plus_description.purpose
+    assert "comprehensive test suites" in plus_description.format
+    assert "edge cases" in plus_description.characteristics
+    assert "evalplus" in plus_description.citation
+
+
+def test_get_evaluation_metrics(parser):
+    """Test evaluation metrics generation for both parsers."""
+    # Test original HumanEval metrics
+    metrics = parser.get_evaluation_metrics()
+    assert len(metrics) == 5  # Base metrics + 2 specific metrics
+
+    # Check primary metrics - update to match actual implementation
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 1  # pass@k
+    assert any(m.name == "pass@k" for m in primary_metrics)

tests/test_ifeval_parser.py

@@ -0,0 +1,120 @@
+import pytest
+
+from llmdataparser.ifeval_parser import IFEvalDatasetParser, IFEvalParseEntry
+
+
+@pytest.fixture
+def sample_ifeval_entries():
+    """Create sample IFEval dataset entries for testing."""
+    return [
+        {
+            "key": 1,
+            "prompt": "Write a function to calculate factorial.",
+            "instruction_id_list": ["math_001", "programming_001"],
+            "kwargs": {"difficulty": "medium", "category": "mathematics"},
+        },
+        {
+            "key": 2,
+            "prompt": "Explain quantum computing.",
+            "instruction_id_list": ["physics_001"],
+            "kwargs": {"difficulty": "hard", "category": "physics"},
+        },
+    ]
+
+
+@pytest.fixture
+def ifeval_parser():
+    """Create an IFEval parser instance."""
+    return IFEvalDatasetParser()
+
+
+def test_ifeval_parse_entry_creation_valid():
+    """Test valid creation of IFEvalParseEntry."""
+    entry = IFEvalParseEntry.create(
+        question="Test instruction",
+        answer="",  # IFEval doesn't have answers
+        raw_question="Test instruction",
+        raw_answer="",
+        key=1,
+        instruction_id_list=["test_001", "test_002"],
+        kwargs={"difficulty": "easy"},
+        task_name="default",
+    )
+
+    assert isinstance(entry, IFEvalParseEntry)
+    assert entry.question == "Test instruction"
+    assert entry.answer == ""
+    assert entry.key == 1
+    assert entry.instruction_id_list == ["test_001", "test_002"]
+    assert entry.kwargs == {"difficulty": "easy"}
+    assert entry.task_name == "default"
+
+
+def test_process_entry_ifeval(ifeval_parser, sample_ifeval_entries):
+    """Test processing entries in IFEval parser."""
+    entry = ifeval_parser.process_entry(sample_ifeval_entries[0])
+
+    assert isinstance(entry, IFEvalParseEntry)
+    assert entry.key == 1
+    assert entry.instruction_id_list == ["math_001", "programming_001"]
+    assert entry.kwargs == {"difficulty": "medium", "category": "mathematics"}
+    assert entry.raw_question == "Write a function to calculate factorial."
+    assert entry.answer == ""  # IFEval doesn't have answers
+    assert entry.task_name == "default"
+
+
+def test_parser_initialization(ifeval_parser):
+    """Test initialization of IFEval parser."""
+    assert ifeval_parser._data_source == "google/IFEval"
+    assert ifeval_parser._default_task == "default"
+    assert ifeval_parser.task_names == ["default"]
+    assert (
+        ifeval_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/google/IFEval"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(ifeval_parser):
+    """Test loading the IFEval dataset."""
+    ifeval_parser.load(split="train")
+    assert ifeval_parser.raw_data is not None
+    assert ifeval_parser.split_names == ["train"]
+    assert ifeval_parser._current_task == "default"
+
+
+def test_parser_string_representation(ifeval_parser):
+    """Test string representation of IFEval parser."""
+    repr_str = str(ifeval_parser)
+    assert "IFEvalDatasetParser" in repr_str
+    assert "google/IFEval" in repr_str
+    assert "not loaded" in repr_str
+
+
+def test_get_dataset_description(ifeval_parser):
+    """Test dataset description generation for IFEval."""
+    description = ifeval_parser.get_dataset_description()
+
+    assert description.name == "IFEval"
+    assert description.source == "Google Research"
+    assert description.language == "English (BCP-47 en)"
+
+
+def test_get_evaluation_metrics(ifeval_parser):
+    """Test evaluation metrics generation for IFEval."""
+    metrics = ifeval_parser.get_evaluation_metrics()
+
+    # Should have 5 metrics total
+    assert len(metrics) == 5
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 3
+
+    # Verify specific metrics exist and have correct properties
+    metric_names = {m.name for m in metrics}
+    assert "format_compliance" in metric_names
+    assert "length_constraints" in metric_names
+    assert "punctuation_rules" in metric_names
+    assert "keyword_usage" in metric_names
+    assert "structural_requirements" in metric_names

tests/test_math_parser.py

@@ -0,0 +1,253 @@
+import pytest
+
+from llmdataparser.math_parser import MATHDatasetParser, MATHParseEntry
+
+
+@pytest.fixture
+def math_parser():
+    """Create a MATH parser instance for testing."""
+    return MATHDatasetParser()
+
+
+@pytest.fixture
+def loaded_math_parser(math_parser):
+    """Create and load a MATH parser instance with test split."""
+    math_parser.load(task_name="algebra", split="test")
+    return math_parser
+
+
+@pytest.fixture
+def sample_math_entries():
+    """Create sample MATH dataset entries for testing."""
+    return [
+        {
+            "problem": "Solve for x: 2x + 4 = 10",
+            "level": "Level 3",
+            "solution": "Let's solve step by step:\n1) Subtract 4 from both sides: 2x = 6\n2) Divide both sides by 2\n\nTherefore, x = 3",
+            "type": "algebra",
+        },
+        {
+            "problem": "Find the area of a circle with radius 5 units.",
+            "level": "Level 2",
+            "solution": "Area = πr²\nArea = π(5)²\nArea = 25π square units",
+            "type": "geometry",
+        },
+        {
+            "problem": "What is the limit of (x²-1)/(x-1) as x approaches 1?",
+            "level": "Level 4",
+            "solution": "Using L'Hôpital's rule:\nlim(x→1) (x²-1)/(x-1) = lim(x→1) (2x)/(1) = 2",
+            "type": "calculus",
+        },
+    ]
+
+
+def test_math_parse_entry_creation_valid():
+    """Test valid creation of MATHParseEntry with all fields."""
+    entry = MATHParseEntry.create(
+        question="Test question",
+        answer="Test answer",
+        raw_question="Test question",
+        raw_answer="Test solution",
+        level="Level 5",
+        task_name="algebra",
+        solution="Test solution",
+    )
+
+    assert isinstance(entry, MATHParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "Test answer"
+    assert entry.raw_question == "Test question"
+    assert entry.raw_answer == "Test solution"
+    assert entry.level == "Level 5"
+    assert entry.task_name == "algebra"
+    assert entry.solution == "Test solution"
+
+
+@pytest.mark.parametrize(
+    "test_case",
+    [
+        {
+            "problem": "Solve for x: 2x + 4 = 10",
+            "level": "Level 3",
+            "solution": "x = 3",
+            "type": "algebra",
+        },
+        {
+            "problem": "Find the derivative of f(x) = x²",
+            "level": "Level 4",
+            "solution": "f'(x) = 2x",
+            "type": "calculus",
+        },
+    ],
+)
+def test_process_entry(math_parser, test_case):
+    """Test processing different types of MATH entries."""
+    entry = math_parser.process_entry(test_case, task_name=test_case["type"])
+
+    assert isinstance(entry, MATHParseEntry)
+
+    assert entry.answer == test_case["solution"]
+    assert entry.raw_question == test_case["problem"]
+    assert entry.raw_answer == test_case["solution"]
+    assert entry.level == test_case["level"]
+    assert entry.task_name == test_case["type"]
+    assert entry.solution == test_case["solution"]
+
+
+def test_math_parser_initialization(math_parser):
+    """Test MATH parser initialization and properties."""
+    assert isinstance(math_parser.task_names, list)
+    assert len(math_parser.task_names) == 8
+    assert math_parser._data_source == "lighteval/MATH"
+    assert math_parser._default_task == "all"
+    assert "algebra" in math_parser.task_names
+    assert "geometry" in math_parser.task_names
+    assert (
+        math_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/lighteval/MATH"
+    )
+
+
+def test_get_current_task(math_parser):
+    """Test task name resolution in different scenarios."""
+    # Test with valid type in data entry
+    test_row_with_type = {"type": "algebra"}
+    assert math_parser._get_current_task(test_row_with_type) == "algebra"
+
+    # Test without type in data entry
+    test_row_without_type = {}
+    math_parser._current_task = "geometry"
+    assert math_parser._get_current_task(test_row_without_type) == "geometry"
+
+    # Test with invalid type - should return current task
+    test_row_invalid_type = {"type": "invalid_type"}
+    math_parser._current_task = "algebra"
+    assert math_parser._get_current_task(test_row_invalid_type) == "algebra"
+
+
+def test_valid_levels(math_parser):
+    """Test handling of valid level values."""
+    for i in range(1, 6):
+        test_row = {
+            "problem": "Test problem",
+            "level": f"Level {i}",
+            "solution": "Test solution",
+            "type": "algebra",
+        }
+        entry = math_parser.process_entry(test_row, task_name="algebra")
+        assert entry.level == f"Level {i}"
+
+
+@pytest.mark.parametrize(
+    "invalid_level",
+    [
+        "Level 0",  # Too low
+        "Level 6",  # Too high
+        "Invalid",  # Wrong format
+        None,  # Missing
+        "",  # Empty
+        "level 1",  # Wrong capitalization
+    ],
+)
+def test_invalid_level_handling(math_parser, invalid_level):
+    """Test handling of invalid level values."""
+    test_row = {
+        "problem": "Test problem",
+        "level": invalid_level,
+        "solution": "Test solution",
+        "type": "algebra",
+    }
+
+    entry = math_parser.process_entry(test_row, task_name="algebra")
+    assert entry.level == "Unknown"
+
+
+@pytest.mark.integration
+def test_load_dataset(loaded_math_parser):
+    """Test loading the MATH dataset."""
+    assert loaded_math_parser.raw_data is not None
+    assert loaded_math_parser.split_names == ["test"]
+    assert loaded_math_parser._current_task == "algebra"
+
+
+def test_parser_string_representation(loaded_math_parser):
+    """Test string representation of MATH parser."""
+    repr_str = str(loaded_math_parser)
+    assert "MATHDatasetParser" in repr_str
+    assert "lighteval/MATH" in repr_str
+    assert "algebra" in repr_str
+    assert "loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_splits_parsing(math_parser):
+    """Test parsing different splits of the dataset."""
+    # Load and parse test split
+    math_parser.load(task_name="algebra", split="test")
+    math_parser.parse(split_names="test", force=True)
+    test_count = len(math_parser.get_parsed_data)
+
+    # Load and parse train split
+    math_parser.load(task_name="algebra", split="train")
+    math_parser.parse(split_names="train", force=True)
+    train_count = len(math_parser.get_parsed_data)
+
+    assert test_count > 0
+    assert train_count > 0
+    assert train_count != test_count
+
+
+def test_get_dataset_description(math_parser):
+    """Test dataset description generation."""
+    description = math_parser.get_dataset_description()
+
+    assert description.name == "MATH"
+    assert "Hendrycks" in description.source
+    assert description.language == "English"
+    assert "12,500" in description.characteristics
+    assert "hendrycksmath2021" in description.citation
+    assert "NeurIPS" in description.citation
+
+    # Check additional info
+    assert description.additional_info is not None
+    assert description.additional_info["difficulty_levels"] == "1-5"
+    assert "algebra" in description.additional_info["topics"]
+    assert "geometry" in description.additional_info["topics"]
+    assert description.additional_info["size"] == "12,500 problems"
+
+
+def test_get_evaluation_metrics(math_parser):
+    """Test evaluation metrics generation."""
+    metrics = math_parser.get_evaluation_metrics()
+
+    # Check total number of metrics
+    assert len(metrics) == 5
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 3
+
+    # Verify specific metrics exist with correct properties
+    metric_names = {m.name for m in metrics}
+    assert "symbolic_equivalence" in metric_names
+    assert "solution_presence" in metric_names
+    assert "reasoning_validity" in metric_names
+    assert "mathematical_notation" in metric_names
+    assert "solution_clarity" in metric_names
+
+    # Check specific metric properties
+    symbolic_metric = next(m for m in metrics if m.name == "symbolic_equivalence")
+    assert symbolic_metric.type == "exact_match"
+    assert symbolic_metric.primary is True
+    assert "sympy" in symbolic_metric.description.lower()
+    assert "equivalence" in symbolic_metric.description.lower()
+
+    solution_metric = next(m for m in metrics if m.name == "solution_presence")
+    assert solution_metric.type == "text"
+    assert solution_metric.primary is True
+    assert "step-by-step" in solution_metric.description.lower()
+
+    reasoning_metric = next(m for m in metrics if m.name == "reasoning_validity")
+    assert reasoning_metric.type == "text"
+    assert reasoning_metric.primary is True
+    assert "mathematical reasoning" in reasoning_metric.description.lower()

tests/test_mbpp_parser.py

@@ -0,0 +1,178 @@
+import pytest
+
+from llmdataparser.mbpp_parser import MBPPDatasetParser, MBPPParseEntry
+
+
+@pytest.fixture
+def sample_entry():
+    return {
+        "text": "Write a function to find the sum of numbers in a list.",
+        "code": "def sum_list(lst):\n    return sum(lst)",
+        "task_id": 42,
+        "test_list": ["assert sum_list([1, 2, 3]) == 6"],
+        "test_setup_code": "",
+        "challenge_test_list": ["assert sum_list([4, 5, 6]) == 15"],
+    }
+
+
+@pytest.fixture
+def parser():
+    return MBPPDatasetParser()
+
+
+def test_mbpp_parse_entry_creation():
+    """Test creation of MBPPParseEntry"""
+    entry = MBPPParseEntry.create(
+        question="test question",
+        answer="test answer",
+        raw_question="raw question",
+        task_id=42,
+        test_list=["test1", "test2"],
+        test_setup_code="setup code",
+        challenge_test_list=["challenge1"],
+        task_name="full",
+        source_file="test.pdf",
+    )
+
+    assert entry.question == "test question"
+    assert entry.answer == "test answer"
+    assert entry.raw_question == "raw question"
+    assert entry.raw_answer == "test answer"
+    assert entry.task_id == 42
+    assert entry.test_list == ["test1", "test2"]
+    assert entry.test_setup_code == "setup code"
+    assert entry.challenge_test_list == ["challenge1"]
+    assert entry.task_name == "full"
+
+
+def test_mbpp_parse_entry_validation():
+    """Test validation of required fields"""
+    with pytest.raises(ValueError, match="Task ID must be an integer"):
+        MBPPParseEntry.create(
+            question="test",
+            answer="test",
+            raw_question="test",
+            task_id="not_an_int",  # Invalid task_id type
+            test_list=[],
+            test_setup_code="",
+            challenge_test_list=[],
+            task_name="full",
+            source_file="test.pdf",
+        )
+
+
+def test_process_entry(parser, sample_entry):
+    """Test processing of a single entry"""
+    result = parser.process_entry(sample_entry, task_name="full")
+
+    assert isinstance(result, MBPPParseEntry)
+    assert result.task_id == 42
+    assert result.raw_question == sample_entry["text"]
+    assert result.answer == sample_entry["code"]
+    assert result.test_list == sample_entry["test_list"]
+    assert result.challenge_test_list == sample_entry["challenge_test_list"]
+    assert result.task_name == "full"
+
+
+def test_parser_initialization(parser):
+    """Test parser initialization and properties"""
+    assert parser._data_source == "google-research-datasets/mbpp"
+    assert parser._default_task == "full"
+    assert parser._task_names == ["full", "sanitized"]
+    assert (
+        parser.get_huggingface_link
+        == "https://huggingface.co/datasets/google-research-datasets/mbpp"
+    )
+
+
+@pytest.mark.integration
+@pytest.mark.skip(reason="Requires access to HuggingFace MBPP dataset")
+def test_parser_load_and_parse(parser):
+    """Integration test for loading and parsing data"""
+    parser.load(split="train")
+    parser.parse(force=True)
+    parsed_data = parser.get_parsed_data
+
+    assert len(parsed_data) > 0
+    assert all(isinstance(entry, MBPPParseEntry) for entry in parsed_data)
+
+
+def test_get_current_task(parser, sample_entry):
+    """Test _get_current_task method"""
+    task = parser._get_current_task(sample_entry)
+    assert task == parser._default_task
+
+
+@pytest.mark.parametrize("task_name", ["full", "sanitized"])
+@pytest.mark.skip(reason="Requires access to HuggingFace MBPP dataset")
+def test_different_tasks_loading(parser, task_name):
+    """Test loading different tasks of the dataset"""
+    parser.load(task_name=task_name, split="train")
+    assert parser._current_task == task_name
+
+
+def test_parser_string_representation(parser):
+    """Test string representation of parser"""
+    repr_str = str(parser)
+    assert "MBPPDatasetParser" in repr_str
+    assert "google-research-datasets/mbpp" in repr_str
+    assert "not loaded" in repr_str
+
+
+def test_parse_without_loaded_data(parser):
+    """Test parsing without loading data first"""
+    with pytest.raises(
+        ValueError, match="No data loaded. Please load the dataset first"
+    ):
+        parser.parse()
+
+
+@pytest.mark.integration
+@pytest.mark.skip(reason="Requires access to HuggingFace MBPP dataset")
+def test_full_workflow_with_different_splits(parser):
+    """Test the complete workflow with different splits"""
+    parser.load(split="train")
+    parser.parse(force=True)
+    train_data = parser.get_parsed_data
+
+    assert len(train_data) > 0
+    assert all(isinstance(entry, MBPPParseEntry) for entry in train_data)
+    assert all(entry.task_name == "full" for entry in train_data)
+
+
+def test_get_dataset_description(parser):
+    """Test dataset description generation."""
+    description = parser.get_dataset_description()
+
+    assert description.name == "Mostly Basic Python Problems (MBPP)"
+    assert "code generation" in description.purpose.lower()
+    assert "google-research" in description.source
+    assert description.language == "English and Python"
+    assert "1,000" in description.characteristics
+    assert "austin2021program" in description.citation
+    assert "Program Synthesis" in description.citation
+
+
+def test_get_evaluation_metrics(parser):
+    """Test evaluation metrics generation."""
+    metrics = parser.get_evaluation_metrics()
+
+    # Check total number of metrics
+    assert len(metrics) == 4
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 1
+
+    # Verify specific metrics exist with correct properties
+    metric_names = {m.name for m in metrics}
+    assert "pass@k" in metric_names
+    assert "test_case_success_rate" in metric_names
+    assert "syntax_validity" in metric_names
+
+    # Check specific metric properties
+    pass_k_metric = next(m for m in metrics if m.name == "pass@k")
+    assert pass_k_metric.type == "code_evaluation"
+    assert pass_k_metric.primary is True
+    assert "k generations" in pass_k_metric.description.lower()
+    assert "custom_pass_at_k" in pass_k_metric.implementation

tests/test_mgsm_parser.py

@@ -0,0 +1,228 @@
+import pytest
+
+from llmdataparser.mgsm_parser import MGSMDatasetParser, MGSMParseEntry
+
+
+@pytest.fixture
+def mgsm_parser():
+    """Create a MGSM parser instance for testing."""
+    return MGSMDatasetParser()
+
+
+@pytest.fixture
+def loaded_mgsm_parser(mgsm_parser):
+    """Create and load a MGSM parser instance with test split."""
+    mgsm_parser.load(task_name="en", split="test")
+    return mgsm_parser
+
+
+@pytest.fixture
+def sample_mgsm_entries():
+    """Create sample MGSM dataset entries for testing."""
+    return [
+        {
+            "question": "John has 5 apples and buys 3 more. How many apples does he have now?",
+            "answer": "Let's solve step by step:\n1) Initial apples = 5\n2) Bought apples = 3\n3) Total = 5 + 3 = 8\nJohn has 8 apples now.",
+            "answer_number": 8,
+            "equation_solution": "5 + 3 = 8",
+            "language": "en",
+        },
+        {
+            "question": "Juan tiene 5 manzanas y compra 3 más. ¿Cuántas manzanas tiene ahora?",
+            "answer": "Resolvamos paso a paso:\n1) Manzanas iniciales = 5\n2) Manzanas compradas = 3\n3) Total = 5 + 3 = 8\nJuan tiene 8 manzanas ahora.",
+            "answer_number": 8,
+            "equation_solution": "5 + 3 = 8",
+            "language": "es",
+        },
+        {
+            "question": "ジョンはリンゴを5個持っていて、さらに3個買います。今何個持っていますか？",
+            "answer": None,  # Testing case with missing detailed answer
+            "answer_number": 8,
+            "equation_solution": "5 + 3 = 8",
+            "language": "ja",
+        },
+    ]
+
+
+def test_mgsm_parse_entry_creation_valid():
+    """Test valid creation of MGSMParseEntry with all fields."""
+    entry = MGSMParseEntry.create(
+        question="Test question",
+        answer="Test answer",
+        raw_question="Test question",
+        raw_answer="Test answer",
+        numerical_answer=42,
+        equation_solution="21 * 2 = 42",
+        task_name="en",
+        language="en",
+    )
+
+    assert isinstance(entry, MGSMParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "Test answer"
+    assert entry.raw_question == "Test question"
+    assert entry.raw_answer == "Test answer"
+    assert entry.numerical_answer == 42
+    assert entry.equation_solution == "21 * 2 = 42"
+    assert entry.task_name == "en"
+    assert entry.language == "en"
+
+
+def test_process_entry_with_detailed_answer(mgsm_parser, sample_mgsm_entries):
+    """Test processing entry with detailed answer in English."""
+    entry = mgsm_parser.process_entry(sample_mgsm_entries[0], task_name="en")
+
+    assert isinstance(entry, MGSMParseEntry)
+    assert entry.numerical_answer == 8
+    assert entry.equation_solution == "5 + 3 = 8"
+    assert "step by step" in entry.answer
+    assert entry.language == "en"
+    assert entry.task_name == "en"
+
+
+def test_process_entry_without_detailed_answer(mgsm_parser, sample_mgsm_entries):
+    """Test processing entry without detailed answer (Japanese)."""
+    entry = mgsm_parser.process_entry(sample_mgsm_entries[2], task_name="ja")
+
+    assert isinstance(entry, MGSMParseEntry)
+    assert entry.numerical_answer == 8
+    assert entry.equation_solution == "5 + 3 = 8"
+    assert entry.answer == "8"  # Should use numerical_answer as string
+    assert entry.language == "ja"
+    assert entry.task_name == "ja"
+
+
+def test_process_entry_spanish(mgsm_parser, sample_mgsm_entries):
+    """Test processing Spanish entry."""
+    entry = mgsm_parser.process_entry(sample_mgsm_entries[1], task_name="es")
+
+    assert isinstance(entry, MGSMParseEntry)
+    assert entry.numerical_answer == 8
+    assert entry.equation_solution == "5 + 3 = 8"
+    assert "paso a paso" in entry.answer  # Spanish for "step by step"
+    assert entry.language == "es"
+    assert entry.task_name == "es"
+
+
+def test_mgsm_parser_initialization(mgsm_parser):
+    """Test MGSM parser initialization and properties."""
+    assert isinstance(mgsm_parser.task_names, list)
+    assert len(mgsm_parser.task_names) == 11  # 11 supported languages
+    assert mgsm_parser._data_source == "juletxara/mgsm"
+    assert mgsm_parser._default_task == "en"
+    assert all(lang in mgsm_parser.task_names for lang in ["en", "es", "ja", "zh"])
+    assert (
+        mgsm_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/juletxara/mgsm"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(loaded_mgsm_parser):
+    """Test loading the MGSM dataset."""
+    assert loaded_mgsm_parser.raw_data is not None
+    assert loaded_mgsm_parser.split_names == ["test"]
+    assert loaded_mgsm_parser._current_task == "en"
+
+
+def test_parser_string_representation(loaded_mgsm_parser):
+    """Test string representation of MGSM parser."""
+    repr_str = str(loaded_mgsm_parser)
+    assert "MGSMDatasetParser" in repr_str
+    assert "juletxara/mgsm" in repr_str
+    assert "en" in repr_str
+    assert "loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_languages_parsing(mgsm_parser):
+    """Test parsing different language versions."""
+    # Load and parse English
+    mgsm_parser.load(task_name="en", split="test")
+    mgsm_parser.parse(split_names="test", force=True)
+    en_count = len(mgsm_parser.get_parsed_data)
+
+    # Load and parse Spanish
+    mgsm_parser.load(task_name="es", split="test")
+    mgsm_parser.parse(split_names="test", force=True)
+    es_count = len(mgsm_parser.get_parsed_data)
+
+    assert en_count > 0
+    assert es_count > 0
+    assert en_count == es_count  # Should have same number of problems in each language
+
+
+@pytest.mark.parametrize("language", ["en", "es", "ja", "zh", "ru"])
+def test_supported_languages(mgsm_parser, language):
+    """Test that each supported language can be processed."""
+    test_entry = {
+        "question": f"Test question in {language}",
+        "answer": f"Test answer in {language}",
+        "answer_number": 42,
+        "equation_solution": "21 * 2 = 42",
+    }
+
+    entry = mgsm_parser.process_entry(test_entry, task_name=language)
+    assert entry.language == language
+    assert entry.task_name == language
+    assert entry.numerical_answer == 42
+
+
+def test_get_dataset_description(mgsm_parser):
+    """Test dataset description generation."""
+    description = mgsm_parser.get_dataset_description()
+
+    assert description.name == "Multilingual Grade School Math (MGSM)"
+    assert "multilingual chain-of-thought reasoning" in description.purpose.lower()
+    assert "juletxara/mgsm" in description.source
+    assert description.language == "Multilingual (11 languages)"
+
+    assert "mathematical reasoning" in description.characteristics.lower()
+
+    # Check citations
+    assert "shi2022language" in description.citation
+    assert "cobbe2021gsm8k" in description.citation
+
+    # Check additional info
+    assert description.additional_info is not None
+    assert len(description.additional_info["languages"]) == 11
+    assert "English" in description.additional_info["languages"]
+    assert "Chinese" in description.additional_info["languages"]
+
+
+def test_get_evaluation_metrics(mgsm_parser):
+    """Test evaluation metrics generation."""
+    metrics = mgsm_parser.get_evaluation_metrics()
+
+    # Check total number of metrics
+    assert len(metrics) == 4
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 3
+
+    # Verify specific metrics exist with correct properties
+    metric_names = {m.name for m in metrics}
+    assert "exact_match" in metric_names
+    assert "solution_validity" in metric_names
+    assert "step_accuracy" in metric_names
+    assert "cross_lingual_consistency" in metric_names
+
+    # Check specific metric properties
+    exact_match_metric = next(m for m in metrics if m.name == "exact_match")
+    assert exact_match_metric.type == "string"
+    assert exact_match_metric.primary is True
+    assert "numerical answers" in exact_match_metric.description.lower()
+    assert "custom_exact_match" in exact_match_metric.implementation
+
+    solution_metric = next(m for m in metrics if m.name == "solution_validity")
+    assert solution_metric.type == "text"
+    assert solution_metric.primary is True
+    assert "mathematically valid" in solution_metric.description.lower()
+    assert "custom_solution_validator" in solution_metric.implementation
+
+    step_metric = next(m for m in metrics if m.name == "step_accuracy")
+    assert step_metric.type == "numerical"
+    assert step_metric.primary is True
+    assert "calculation steps" in step_metric.description.lower()
+    assert "custom_step_accuracy" in step_metric.implementation

tests/test_mmlu_parser.py

@@ -0,0 +1,314 @@
+import pytest
+
+from llmdataparser.mmlu_parser import (
+    BaseMMLUDatasetParser,
+    MMLUParseEntry,
+    MMLUProDatasetParser,
+    MMLUProParseEntry,
+    MMLUReduxDatasetParser,
+    TMMLUPlusDatasetParser,
+)
+
+
+@pytest.fixture
+def base_parser():
+    """Create a base MMLU parser instance."""
+    return BaseMMLUDatasetParser()
+
+
+@pytest.fixture
+def redux_parser():
+    """Create a MMLU Redux parser instance."""
+    return MMLUReduxDatasetParser()
+
+
+@pytest.fixture
+def tmmlu_parser():
+    """Create a TMMLU+ parser instance."""
+    return TMMLUPlusDatasetParser()
+
+
+@pytest.fixture
+def mmlu_pro_parser():
+    """Create a MMLU Pro parser instance."""
+    return MMLUProDatasetParser()
+
+
+@pytest.fixture
+def sample_mmlu_entries():
+    """Create sample MMLU dataset entries for testing."""
+    return [
+        {
+            "question": "What is the capital of France?",
+            "choices": ["London", "Paris", "Berlin", "Madrid"],
+            "answer": 1,  # Paris
+            "subject": "geography",
+        },
+        {
+            "question": "Which of these is a primary color?",
+            "choices": ["Green", "Purple", "Blue", "Orange"],
+            "answer": 2,  # Blue
+            "subject": "art",
+        },
+    ]
+
+
+@pytest.fixture
+def sample_mmlu_pro_entries():
+    """Create sample MMLU Pro dataset entries for testing."""
+    return [
+        {
+            "question": "What is the time complexity of quicksort?",
+            "options": ["O(n)", "O(n log n)", "O(n²)", "O(2ⁿ)", "O(n!)", "O(1)"],
+            "answer": "The average time complexity of quicksort is O(n log n)",
+            "answer_index": 1,
+            "category": "computer_science",
+        }
+    ]
+
+
+def test_mmlu_parse_entry_creation_valid():
+    """Test valid creation of MMLUParseEntry."""
+    entry = MMLUParseEntry.create(
+        question="Test question",
+        answer="A",
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4"],
+        raw_answer="0",
+        task_name="test_task",
+    )
+    assert isinstance(entry, MMLUParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["choice1", "choice2", "choice3", "choice4"]
+    assert entry.task_name == "test_task"
+
+
+@pytest.mark.parametrize("invalid_answer", ["E", "F", "1", "", None])
+def test_mmlu_parse_entry_creation_invalid(invalid_answer):
+    """Test invalid answer handling in MMLUParseEntry creation."""
+    with pytest.raises(
+        ValueError, match="Invalid answer_letter.*must be one of A, B, C, D"
+    ):
+        MMLUParseEntry.create(
+            question="Test question",
+            answer=invalid_answer,
+            raw_question="Test question",
+            raw_choices=["choice1", "choice2", "choice3", "choice4"],
+            raw_answer="4",
+            task_name="test_task",
+        )
+
+
+def test_process_entry_base(base_parser, sample_mmlu_entries):
+    """Test processing entries in base MMLU parser."""
+    entry = base_parser.process_entry(sample_mmlu_entries[0], task_name="geography")
+
+    assert isinstance(entry, MMLUParseEntry)
+    assert entry.answer == "B"  # Index 1 maps to B
+    assert "A. London" in entry.question
+    assert "B. Paris" in entry.question
+    assert "C. Berlin" in entry.question
+    assert "D. Madrid" in entry.question
+    assert entry.raw_question == "What is the capital of France?"
+    assert entry.raw_choices == ["London", "Paris", "Berlin", "Madrid"]
+    assert entry.raw_answer == "1"
+    assert entry.task_name == "geography"
+
+
+def test_mmlu_pro_parse_entry_creation_valid():
+    """Test valid creation of MMLUProParseEntry."""
+    entry = MMLUProParseEntry.create(
+        question="Test question",
+        answer="E",  # MMLU Pro supports up to J
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4", "choice5"],
+        raw_answer="4",
+        task_name="test_task",
+    )
+    assert isinstance(entry, MMLUProParseEntry)
+    assert entry.answer == "E"
+    assert len(entry.raw_choices) == 5
+
+
+def test_process_entry_mmlu_pro(mmlu_pro_parser, sample_mmlu_pro_entries):
+    """Test processing entries in MMLU Pro parser."""
+    entry = mmlu_pro_parser.process_entry(
+        sample_mmlu_pro_entries[0], task_name="computer_science"
+    )
+
+    assert isinstance(entry, MMLUProParseEntry)
+    assert entry.answer == "B"  # Index 1 maps to B
+    assert "O(n log n)" in entry.question
+    assert entry.task_name == "computer_science"
+    assert len(entry.raw_choices) == 6
+
+
+def test_tmmlu_process_entry(tmmlu_parser):
+    """Test processing entries in TMMLU+ parser."""
+    test_row = {
+        "question": "什麼是台灣最高的山峰？",
+        "A": "玉山",
+        "B": "阿里山",
+        "C": "合歡山",
+        "D": "雪山",
+        "answer": "A",
+        "subject": "geography_of_taiwan",
+    }
+
+    entry = tmmlu_parser.process_entry(test_row, task_name="geography_of_taiwan")
+    assert isinstance(entry, MMLUParseEntry)
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["玉山", "阿里山", "合歡山", "雪山"]
+    assert entry.task_name == "geography_of_taiwan"
+
+
+@pytest.mark.parametrize(
+    "parser_fixture,expected_tasks,expected_source",
+    [
+        ("base_parser", 57, "cais/mmlu"),
+        ("redux_parser", 30, "edinburgh-dawg/mmlu-redux"),
+        ("tmmlu_parser", 66, "ikala/tmmluplus"),
+        ("mmlu_pro_parser", 1, "TIGER-Lab/MMLU-Pro"),
+    ],
+)
+def test_parser_initialization(
+    request, parser_fixture, expected_tasks, expected_source
+):
+    """Test initialization of different MMLU parser variants."""
+    parser = request.getfixturevalue(parser_fixture)
+    assert len(parser.task_names) == expected_tasks
+    assert parser._data_source == expected_source
+    assert (
+        parser.get_huggingface_link
+        == f"https://huggingface.co/datasets/{expected_source}"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(base_parser):
+    """Test loading the MMLU dataset."""
+    base_parser.load(task_name="anatomy", split="test")
+    assert base_parser.raw_data is not None
+    assert base_parser.split_names == ["test"]
+    assert base_parser._current_task == "anatomy"
+
+
+def test_parser_string_representation(base_parser):
+    """Test string representation of MMLU parser."""
+    repr_str = str(base_parser)
+    assert "MMLUDatasetParser" in repr_str
+    assert "cais/mmlu" in repr_str
+    assert "not loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_splits_parsing(base_parser):
+    """Test parsing different splits of the dataset."""
+    # Load and parse test split
+    base_parser.load(task_name="anatomy", split="test")
+    base_parser.parse(split_names="test", force=True)
+    test_count = len(base_parser.get_parsed_data)
+
+    # Load and parse validation split
+    base_parser.load(task_name="anatomy", split="validation")
+    base_parser.parse(split_names="validation", force=True)
+    val_count = len(base_parser.get_parsed_data)
+
+    assert test_count > 0
+    assert val_count > 0
+    assert test_count != val_count
+
+
+def test_base_mmlu_dataset_description(base_parser):
+    """Test dataset description for base MMLU."""
+    description = base_parser.get_dataset_description()
+
+    assert description.name == "Massive Multitask Language Understanding (MMLU)"
+    assert "cais/mmlu" in description.source
+    assert description.language == "English"
+
+    # Check characteristics
+    assert "57 subjects" in description.characteristics.lower()
+
+    # Check citation
+    assert "hendryckstest2021" in description.citation
+
+
+def test_mmlu_redux_dataset_description(redux_parser):
+    """Test dataset description for MMLU Redux."""
+    description = redux_parser.get_dataset_description()
+
+    assert description.name == "MMLU Redux"
+    assert "manually re-annotated" in description.purpose.lower()
+    assert "edinburgh-dawg/mmlu-redux" in description.source
+    assert description.language == "English"
+
+    # Check characteristics
+    assert "3,000" in description.characteristics
+
+
+def test_tmmlu_plus_dataset_description(tmmlu_parser):
+    """Test dataset description for TMMLU+."""
+    description = tmmlu_parser.get_dataset_description()
+
+    assert "ikala/tmmluplus" in description.source
+    assert description.language == "Traditional Chinese"
+
+    # Check characteristics
+    assert "66 subjects" in description.characteristics.lower()
+
+    # Check citation
+    assert "ikala2024improved" in description.citation
+
+
+def test_mmlu_pro_dataset_description(mmlu_pro_parser):
+    """Test dataset description for MMLU Pro."""
+    description = mmlu_pro_parser.get_dataset_description()
+
+    assert description.name == "MMLU Pro"
+    assert "challenging" in description.purpose.lower()
+    assert "TIGER-Lab/MMLU-Pro" in description.source
+    assert description.language == "English"
+
+
+def test_base_mmlu_evaluation_metrics(base_parser):
+    """Test evaluation metrics for base MMLU."""
+    metrics = base_parser.get_evaluation_metrics()
+
+    assert len(metrics) >= 3
+    metric_names = {m.name for m in metrics}
+
+    assert "accuracy" in metric_names
+    assert "subject_accuracy" in metric_names
+    assert "category_accuracy" in metric_names
+
+    accuracy_metric = next(m for m in metrics if m.name == "accuracy")
+    assert accuracy_metric.type == "classification"
+    assert accuracy_metric.primary is True
+    assert "multiple-choice" in accuracy_metric.description.lower()
+
+
+def test_mmlu_redux_evaluation_metrics(redux_parser):
+    """Test evaluation metrics for MMLU Redux."""
+    metrics = redux_parser.get_evaluation_metrics()
+
+    metric_names = {m.name for m in metrics}
+    assert "question_clarity" in metric_names
+
+
+def test_tmmlu_plus_evaluation_metrics(tmmlu_parser):
+    """Test evaluation metrics for TMMLU+."""
+    metrics = tmmlu_parser.get_evaluation_metrics()
+
+    metric_names = {m.name for m in metrics}
+    assert "difficulty_analysis" in metric_names
+
+
+def test_mmlu_pro_evaluation_metrics(mmlu_pro_parser):
+    """Test evaluation metrics for MMLU Pro."""
+    metrics = mmlu_pro_parser.get_evaluation_metrics()
+
+    metric_names = {m.name for m in metrics}
+    assert "reasoning_analysis" in metric_names
+    assert "prompt_robustness" in metric_names

tests/test_tmlu_parser.py

@@ -0,0 +1,176 @@
+import pytest
+
+from llmdataparser.tmlu_parser import TMLUDatasetParser, TMLUParseEntry
+
+
+@pytest.fixture
+def tmlu_parser():
+    """Create a TMLU parser instance for testing."""
+    return TMLUDatasetParser()
+
+
+@pytest.fixture
+def sample_tmlu_entries():
+    """Create sample TMLU dataset entries for testing."""
+    return [
+        {
+            "question": "閱讀下文，選出依序最適合填入□內的選項：",
+            "A": "張揚／綢繆未雨／奏疏",
+            "B": "抽搐／煮繭抽絲／奏疏",
+            "C": "張揚／煮繭抽絲／進貢",
+            "D": "抽搐／綢繆未雨／進貢",
+            "answer": "B",
+            "explanation": "根據文意，選項B最為恰當。",
+            "metadata": {
+                "timestamp": "2023-10-09T18:27:20.304623",
+                "source": "AST chinese - 108",
+                "explanation_source": "",
+            },
+        },
+        {
+            "question": "下列何者是質數？",
+            "A": "21",
+            "B": "27",
+            "C": "31",
+            "D": "33",
+            "answer": "C",
+            "explanation": "31是質數，其他選項都是合數。",
+            "metadata": {
+                "timestamp": "2023-10-09T18:27:20.304623",
+                "source": "AST mathematics - 108",
+                "explanation_source": "",
+            },
+        },
+    ]
+
+
+def test_tmlu_parse_entry_creation_valid():
+    """Test valid creation of TMLUParseEntry."""
+    entry = TMLUParseEntry.create(
+        question="Test question",
+        answer="A",
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4"],
+        raw_answer="A",
+        task_name="AST_chinese",
+        explanation="Test explanation",
+        metadata={"source": "test"},
+    )
+    assert isinstance(entry, TMLUParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["choice1", "choice2", "choice3", "choice4"]
+    assert entry.explanation == "Test explanation"
+    assert entry.metadata == {"source": "test"}
+
+
+@pytest.mark.parametrize("invalid_answer", ["E", "F", "1", "", None])
+def test_tmlu_parse_entry_creation_invalid(invalid_answer):
+    """Test invalid answer handling in TMLUParseEntry creation."""
+    with pytest.raises(
+        ValueError, match="Invalid answer_letter.*must be one of A, B, C, D"
+    ):
+        TMLUParseEntry.create(
+            question="Test question",
+            answer=invalid_answer,
+            raw_question="Test question",
+            raw_choices=["choice1", "choice2", "choice3", "choice4"],
+            raw_answer=invalid_answer,
+            task_name="AST_chinese",
+        )
+
+
+def test_process_entry(tmlu_parser, sample_tmlu_entries):
+    """Test processing entries in TMLU parser."""
+    entry = tmlu_parser.process_entry(sample_tmlu_entries[0], task_name="AST_chinese")
+
+    assert isinstance(entry, TMLUParseEntry)
+    assert entry.answer == "B"
+    assert entry.task_name == "AST_chinese"
+    assert len(entry.raw_choices) == 4
+    assert entry.explanation == "根據文意，選項B最為恰當。"
+    assert "AST chinese - 108" in entry.metadata["source"]
+
+
+def test_tmlu_parser_initialization(tmlu_parser):
+    """Test TMLU parser initialization and properties."""
+    assert isinstance(tmlu_parser.task_names, list)
+    assert len(tmlu_parser.task_names) == 37  # Total number of tasks
+    assert tmlu_parser._data_source == "miulab/tmlu"
+    assert tmlu_parser._default_task == "AST_chinese"
+    assert "AST_chinese" in tmlu_parser.task_names
+    assert "GSAT_mathematics" in tmlu_parser.task_names
+    assert (
+        tmlu_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/miulab/tmlu"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(tmlu_parser):
+    """Test loading the TMLU dataset."""
+    tmlu_parser.load(task_name="AST_chinese", split="test")
+    assert tmlu_parser.raw_data is not None
+    assert tmlu_parser.split_names == ["test"]
+    assert tmlu_parser._current_task == "AST_chinese"
+
+
+def test_parser_string_representation(tmlu_parser):
+    """Test string representation of TMLU parser."""
+    repr_str = str(tmlu_parser)
+    assert "TMLUDatasetParser" in repr_str
+    assert "miulab/tmlu" in repr_str
+    assert "not loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_different_tasks_parsing(tmlu_parser):
+    """Test parsing different tasks of the dataset."""
+    # Load and parse AST_chinese
+    tmlu_parser.load(task_name="AST_chinese", split="test")
+    tmlu_parser.parse(split_names="test", force=True)
+    chinese_count = len(tmlu_parser.get_parsed_data)
+
+    # Load and parse AST_mathematics
+    tmlu_parser.load(task_name="AST_mathematics", split="test")
+    tmlu_parser.parse(split_names="test", force=True)
+    math_count = len(tmlu_parser.get_parsed_data)
+
+    assert chinese_count > 0
+    assert math_count > 0
+
+
+def test_metadata_handling(tmlu_parser, sample_tmlu_entries):
+    """Test proper handling of metadata in entries."""
+    entry = tmlu_parser.process_entry(sample_tmlu_entries[0])
+
+    assert "timestamp" in entry.metadata
+    assert "source" in entry.metadata
+    assert "explanation_source" in entry.metadata
+    assert entry.metadata["source"] == "AST chinese - 108"
+
+
+def test_get_dataset_description(tmlu_parser):
+    """Test dataset description generation."""
+    description = tmlu_parser.get_dataset_description()
+
+    assert description.name == "Taiwan Multiple-choice Language Understanding (TMLU)"
+    assert description.language == "Traditional Chinese"
+    assert "Taiwan-specific educational" in description.purpose
+    assert "Various Taiwan standardized tests" in description.source
+    assert description.format == "Multiple choice questions (A/B/C/D)"
+    assert "Advanced Subjects Test (AST)" in description.characteristics
+    assert "DBLP:journals/corr/abs-2403-20180" in description.citation
+
+
+def test_get_evaluation_metrics(tmlu_parser):
+    """Test evaluation metrics generation."""
+    metrics = tmlu_parser.get_evaluation_metrics()
+
+    assert len(metrics) == 2  # Check total number of metrics
+
+    # Check primary metrics
+    primary_metrics = [m for m in metrics if m.primary]
+    assert len(primary_metrics) == 2
+    assert any(m.name == "accuracy" for m in primary_metrics)
+    assert any(m.name == "per_subject_accuracy" for m in primary_metrics)

tests/test_tw_legal_parser.py

@@ -0,0 +1,146 @@
+import pytest
+
+from llmdataparser.tw_legal_parser import TWLegalDatasetParser, TWLegalParseEntry
+
+
+@pytest.fixture
+def tw_legal_parser():
+    """Create a Taiwan Legal parser instance for testing."""
+    return TWLegalDatasetParser()
+
+
+@pytest.fixture
+def sample_tw_legal_entries():
+    """Create sample Taiwan Legal dataset entries for testing."""
+    return [
+        {
+            "question": "依民法規定，下列關於法人之敘述，何者錯誤？",
+            "A": "法人於法令限制內，有享受權利負擔義務之能力",
+            "B": "法人因目的之達到而消滅",
+            "C": "法人非依法律之規定，不得成立",
+            "D": "法人於登記前，即取得權利能力",
+            "answer": "D",
+        },
+        {
+            "question": "關於刑法第321條第1項第4款之結夥三人以上而犯竊盜罪，下列敘述何者正確？",
+            "A": "須行為人主觀上有結夥犯竊盜之認識",
+            "B": "三人以上當場在場實施竊盜行為始足當之",
+            "C": "三人以上已達成犯意聯絡即可成立",
+            "D": "三人以上須全部在現場實施竊盜行為",
+            "answer": "A",
+        },
+    ]
+
+
+def test_tw_legal_parse_entry_creation_valid():
+    """Test valid creation of TWLegalParseEntry."""
+    entry = TWLegalParseEntry.create(
+        question="Test question",
+        answer="A",
+        raw_question="Test question",
+        raw_choices=["choice1", "choice2", "choice3", "choice4"],
+        raw_answer="A",
+        task_name="default",
+    )
+    assert isinstance(entry, TWLegalParseEntry)
+    assert entry.question == "Test question"
+    assert entry.answer == "A"
+    assert entry.raw_choices == ["choice1", "choice2", "choice3", "choice4"]
+
+
+@pytest.mark.parametrize("invalid_answer", ["E", "F", "1", "", None])
+def test_tw_legal_parse_entry_creation_invalid(invalid_answer):
+    """Test invalid answer handling in TWLegalParseEntry creation."""
+    with pytest.raises(
+        ValueError, match="Invalid answer_letter.*must be one of A, B, C, D"
+    ):
+        TWLegalParseEntry.create(
+            question="Test question",
+            answer=invalid_answer,
+            raw_question="Test question",
+            raw_choices=["choice1", "choice2", "choice3", "choice4"],
+            raw_answer=invalid_answer,
+            task_name="default",
+        )
+
+
+def test_process_entry(tw_legal_parser, sample_tw_legal_entries):
+    """Test processing entries in Taiwan Legal parser."""
+    entry = tw_legal_parser.process_entry(sample_tw_legal_entries[0])
+
+    assert isinstance(entry, TWLegalParseEntry)
+    assert entry.answer == "D"
+    assert "A. 法人於法令限制內，有享受權利負擔義務之能力" in entry.question
+    assert "B. 法人因目的之達到而消滅" in entry.question
+    assert "C. 法人非依法律之規定，不得成立" in entry.question
+    assert "D. 法人於登記前，即取得權利能力" in entry.question
+    assert entry.raw_question == "依民法規定，下列關於法人之敘述，何者錯誤？"
+    assert len(entry.raw_choices) == 4
+
+
+def test_tw_legal_parser_initialization(tw_legal_parser):
+    """Test Taiwan Legal parser initialization and properties."""
+    assert isinstance(tw_legal_parser.task_names, list)
+    assert len(tw_legal_parser.task_names) == 1  # Only default task
+    assert tw_legal_parser._data_source == "lianghsun/tw-legal-benchmark-v1"
+    assert tw_legal_parser._default_task == "default"
+    assert (
+        tw_legal_parser.get_huggingface_link
+        == "https://huggingface.co/datasets/lianghsun/tw-legal-benchmark-v1"
+    )
+
+
+@pytest.mark.integration
+def test_load_dataset(tw_legal_parser):
+    """Test loading the Taiwan Legal dataset."""
+    tw_legal_parser.load(split="train")
+    assert tw_legal_parser.raw_data is not None
+    assert tw_legal_parser.split_names == ["train"]
+    assert tw_legal_parser._current_task == "default"
+
+
+def test_parser_string_representation(tw_legal_parser):
+    """Test string representation of Taiwan Legal parser."""
+    repr_str = str(tw_legal_parser)
+    assert "TWLegalDatasetParser" in repr_str
+    assert "lianghsun/tw-legal-benchmark-v1" in repr_str
+    assert "not loaded" in repr_str
+
+
+@pytest.mark.integration
+def test_data_parsing(tw_legal_parser):
+    """Test parsing the dataset."""
+    # Load and parse train split
+    tw_legal_parser.load(split="train")
+    tw_legal_parser.parse(split_names="train", force=True)
+    train_count = len(tw_legal_parser.get_parsed_data)
+
+    assert train_count > 0
+    # Additional assertions about the parsed data
+    parsed_data = tw_legal_parser.get_parsed_data
+    assert all(isinstance(entry, TWLegalParseEntry) for entry in parsed_data)
+    assert all(entry.answer in {"A", "B", "C", "D"} for entry in parsed_data)
+
+
+def test_get_dataset_description(tw_legal_parser):
+    """Test getting dataset description for Taiwan Legal parser."""
+    description = tw_legal_parser.get_dataset_description()
+
+    assert description.name == "Taiwan Legal Benchmark"
+    assert description.language == "Traditional Chinese"
+    assert "Taiwan's legal system" in description.characteristics
+    assert (
+        "huggingface.co/datasets/lianghsun/tw-legal-benchmark-v1"
+        in description.citation
+    )
+
+
+def test_get_evaluation_metrics(tw_legal_parser):
+    """Test getting evaluation metrics for Taiwan Legal parser."""
+    metrics = tw_legal_parser.get_evaluation_metrics()
+
+    assert len(metrics) == 1
+    metric = metrics[0]
+    assert metric.name == "accuracy"
+    assert metric.type == "classification"
+    assert metric.primary is True