LR2Bench

Running

File size: 15,978 Bytes

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>title</title>
<style>
    :root {
        --primary-color: #2c3e50; /* Dark blue-gray for sophistication */
        --secondary-color: #34495e; /* Complementary darker shade */
        --background-color: #f5f5f5; /* Light gray for a clean look */
        --text-color: #2c3e50; /* Dark text for readability */
        --accent-color: #e74c3c; /* Elegant accent for highlights */
        --header-text-color: #ffffff; /* White for table headers */
    }
    body {
        font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
        line-height: 1.6;
        color: var(--text-color);
        max-width: 1200px;
        margin: 0 auto;
        padding: 20px;
        background-color: var(--background-color);
    }
    h1 {
        color: var(--primary-color);
        text-align: center;
        font-size: 2rem;
        margin-bottom: 30px;
    }
    .tab {
        display: flex;
        justify-content: center;
        margin-bottom: 20px;
    }
    .tab button {
        background-color: var(--primary-color);
        color: var(--header-text-color);
        border: none;
        outline: none;
        cursor: pointer;
        padding: 12px 18px;
        font-size: 14px;
        border-radius: 5px;
        margin: 0 5px;
        transition: background-color 0.3s ease, transform 0.2s ease;
    }
    .tab button:hover {
        background-color: var(--secondary-color);
        transform: scale(1.05);
    }
    .tab button.active {
        background-color: var(--accent-color);
    }
    #taskDescription {
        text-align: left;
        font-size: 1rem;
        margin-bottom: 20px;
        color: var(--text-color);
    }
    .tabcontent {
        display: none;
        padding: 20px;
        background-color: white;
        border-radius: 5px;
        box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
    }
    table {
        border-collapse: collapse;
        width: 100%;
        margin-bottom: 20px;
        background-color: white;
        border-radius: 5px;
        overflow: hidden;
        box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
    }
    th, td {
        text-align: left;
        padding: 12px;
        font-size: 14px;
        color: var(--text-color);
        border-bottom: 1px solid #ddd;
    }
    th {
        background-color: var(--primary-color);
        color: var(--header-text-color);
        font-size: 15px;
        font-weight: bold;
    }
tr:nth-child(odd) {
    background-color: #fdfdfd; /* Light background for odd rows */
}
tr:nth-child(even) {
    background-color: #f7f9fc; /* Slightly darker background for even rows */
}
/* 表格行悬停样式 */
tr:hover {
    background-color: var(--secondary-color);
}
/* 单元格悬停时文本颜色 */
tr:hover th, tr:hover td {
    color: var(--header-text-color); /* 确保文本颜色为白色 */
}
/* 悬停时链接的颜色 */
tr:hover a {
    color: inherit; /* 继承自父元素的颜色 */
}
    a {
        color: var(--accent-color);
        text-decoration: none;
        font-weight: bold;
    }
    a:hover {
        text-decoration: underline;
    }
    th a {
        color: inherit;
        display: block;
        text-align: left;
        font-size: inherit;
        padding: 0;
    }
    th:hover, th a:hover {
        background-color: var(--accent-color);
    }
</style>

<script type="text/javascript" async
  src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.7/MathJax.js?config=TeX-MML-AM_CHTML">
</script>
  <script type="text/javascript">
    window.onload = function() {
      MathJax.Hub.Config({
        tex2jax: {
          inlineMath: [['$', '$'], ['\\(', '\\)']],
          displayMath: [['$$', '$$'], ['\\[', '\\]']]
        },
        TeX: {
          extensions: ['AMSmath.js', 'AMSsymbols.js']
        }
      });

      MathJax.Hub.Queue(["Typeset", MathJax.Hub]);
    };
  </script>
</head>
<body>
    <h1>LR\({}^{2}\)Bench: Evaluating Long-chain Reflective Reasoning Capabilities of Large Language Models via Constraint Satisfaction Problems</h1>
    <p>
        <strong>LR\({}^{2}\)Bench</strong> is a novel benchmark designed to evaluate the <strong>L</strong>ong-chain <strong>R</strong>eflective <strong>R</strong>easoning capabilities of LLMs. LR\({}^{2}\)Bench comprises 850 samples across six Constraint Satisfaction Problems (CSPs) where reflective reasoning is crucial for deriving solutions that meet all given constraints. Each type of task focuses on distinct constraint patterns, such as knowledge-based, logical, and spatial constraints, providing a comprehensive evaluation of diverse problem-solving scenarios.
    </p>
    <hr />

    <div class="tab">
        <button class="tablinks" onclick="openTab(event, 'Acrostic')" id="defaultOpen">Acrostic</button>
        <button class="tablinks" onclick="openTab(event, 'Crossword')">Crossword</button>
        <button class="tablinks" onclick="openTab(event, 'Cryptogram')">Cryptogram</button>
        <button class="tablinks" onclick="openTab(event, 'Logic Puzzle')">Logic Puzzle</button>
        <button class="tablinks" onclick="openTab(event, 'Sudoku')">Sudoku</button>
        <button class="tablinks" onclick="openTab(event, 'Drop Quote')">Drop Quote</button>
         
    </div>
    <div id="taskDescription"></div>

  
    <div id="Acrostic" class="tabcontent"></div>
    <div id="Drop Quote" class="tabcontent"></div>
    <div id="Crossword" class="tabcontent"></div>
    <div id="Logic Puzzle" class="tabcontent"></div>
    <div id="Cryptogram" class="tabcontent"></div>
    <div id="Sudoku" class="tabcontent"></div>
    
    <script>
        const descriptions = {
            'Acrostic': 'The Acrostic task involves word clues like Crossword, but its objective is to form a hidden quotation or sentence from the answers to the clues. This requires that the answer words not only satisfy the corresponding clues but also effectively integrate to construct the ultimate hidden message. We collected 50 easy and 50 hard Acrostic samples from <a href="https://www.printable-puzzles.com/printable-acrostic-puzzles.php" target="_blank"> Printable Puzzles</a> with timestamps ranging from September 2024 to December 2024.',
            'Crossword': 'The Crossword task requires inferring correct words from given clues and filling them into a grid. A key challenge lies in satisfying the constraint of shared letter intersections between horizontal and vertical words. We collected 150 Crossword samples published in 2024 from <a href="https://www.latimes.com" target="_blank"> Los Angeles Times</a> and <a href="https://www.vulture.com" target="_blank"> Vulture</a> in three sizes: $5\\times5$, $10\\times10$, and $15\\times15$, with 50 ones for each size.',
            'Logic Puzzle': 'The Logic Puzzle task constitutes a problem that necessitates logical reasoning to deduce relationships between a set of entities based on the given constraints and clues. The objective is to systematically analyze the given information, employing techniques such as hypothesis formation, elimination, and deductive inference, to determine a unique solution that satisfies all given constraints. We collected 50 puzzles for each of the four sizes ($4\\times4$, $4\\times5$, $4\\times6$, and $4\\times7$) from <a href="https://www.printable-puzzles.com/printable-logic-puzzles.php" target="_blank"> Printable Puzzles</a>, with timestamps ranging from September 2024 to December 2024.',
            'Cryptogram': 'The Cryptogram task involves the decryption of an encrypted quotation or sentence, where each letter of an original text is substituted with another, resulting in an apparently nonsense text. Decryption requires identifying patterns, common letter frequencies, and word structures to deduce the letter-to-letter correspondences, ultimately reconstructing the original content. We collected 50 easy and 50 hard samples from <a href="https://www.printable-puzzles.com/printable-cryptograms.php" target="_blank"> Printable Puzzles</a> with timestamps ranging from September 2024 to December 2024.',
            'Sudoku': 'The Sudoku task consists of filling a \\( n^2 \\times n^2 \\) grid with digits from \\( 1 \\) to \\( n^2 \\), subject to the constraint that each row, column, and \\( n \\times n \\) subgrid contains all digits from 1 to \\( n^2 \\) without repetition. Success in Sudoku relies on logical deduction and careful consideration of the existing digits to determine valid placements for the remaining numbers. From <a href="https://1sudoku.com" target="_blank"> 1sudoku</a>, we collected 200 Sudoku samples in total: 50 easy and 50 hard samples for both \\( 4\\times4 \\) and \\( 9\\times9 \\) sizes.',
            'Drop Quote': 'The Drop Quote task comprises a grid of multiple rows and columns, with each column providing a set of candidate letters. The task requires determining the correct row for letters in each column, effectively "dropping" it into target place to reveal the hidden quotation. We created 50 easy samples by manually compiling common quotations, and collected 50 hard samples from <a href="https://www.printable-puzzles.com/printable-drop-quotes.php" target="_blank"> Printable Puzzles</a>, with timestamps ranging from September 2024 to December 2024.'
        };
        function openTab(evt, tabName) {
            var i, tabcontent, tablinks;
            tabcontent = document.getElementsByClassName("tabcontent");
            for (i = 0; i < tabcontent.length; i++) {
                tabcontent[i].style.display = "none";
            }
            tablinks = document.getElementsByClassName("tablinks");
            for (i = 0; i < tablinks.length; i++) {
                tablinks[i].className = tablinks[i].className.replace(" active", "");
            }
            document.getElementById(tabName).style.display = "block";
            evt.currentTarget.className += " active";

            const descriptionElement = document.getElementById("taskDescription");
            descriptionElement.innerHTML = descriptions[tabName] || "Select a task to see its description.";
            MathJax.Hub.Queue(["Typeset", MathJax.Hub, "taskDescription"]);
        }
    
        document.getElementById("defaultOpen").click();
    
        const urls = {
            'Acrostic': 'Acrostic.json',
            'Drop Quote': 'Drop_Quote.json',
            'Crossword': 'Crossword.json',
            'Sudoku': 'Sudoku.json',
            'Logic Puzzle': 'Logic_Puzzle.json',
            'Cryptogram': 'Cryptogram.json',
        };

        const modelurls = {
            'Llama-3.1-8B-Instruct': 'https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct',
            'Llama-3.1-70B-Instruct': 'https://huggingface.co/meta-llama/Llama-3.1-70B-Instruct',
            'Llama-3.3-70B-Instruct': 'https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct',
            'Mistral-7B-Instruct-v0.3': 'https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3',
            'Mistral-Small-Instruct-2409': 'https://huggingface.co/mistralai/Mistral-Small-Instruct-2409',
            'Mistral-Large-Instruct-2411': 'https://huggingface.co/mistralai/Mistral-Large-Instruct-2411',
            'Qwen2.5-7B-Instruct': 'https://huggingface.co/Qwen/Qwen2.5-7B-Instruct',
            'Qwen2.5-32B-Instruct': 'https://huggingface.co/Qwen/Qwen2.5-32B-Instruct',
            'Qwen2.5-72B-Instruct': 'https://huggingface.co/Qwen/Qwen2.5-72B-Instruct',
            'QwQ-32B-Preview': 'https://huggingface.co/Qwen/QwQ-32B-Preview',
            'DeepSeek-R1': 'https://huggingface.co/deepseek-ai/DeepSeek-R1'

        }
    
        function createTable(data, tableId) {
            let table = `<table id="${tableId}">`;
            table += '<thead><tr>';
            for (let key in data[0]) {
                table += `<th><a href="javascript:void(0);" onclick="sortTable('${tableId}', ${Object.keys(data[0]).indexOf(key)})">${key}</a></th>`;
            }
            table += '</tr></thead><tbody>';
            data.forEach(row => {
                table += '<tr>';
                for (let key in row) {
                    if (key === 'Model') {
                        const link = modelurls[row[key]];
                        if (link) {
                            row[key] = `<a href="${link}" target="_blank" rel="noopener noreferrer">${row[key]}</a>`;
                        }
                    }
                    table += `<td>${row[key]}</td>`;
                }
                table += '</tr>';
            });
            table += '</tbody></table>';
            return table;
        }
    
        function sortTable(tableId, n) {
            var table, rows, switching, i, x, y, shouldSwitch, dir, switchcount = 0;
            table = document.getElementById(tableId);
            switching = true;
            dir = "asc";
            while (switching) {
                switching = false;
                rows = table.rows;
                for (i = 1; i < (rows.length - 1); i++) {
                    shouldSwitch = false;
                    x = rows[i].getElementsByTagName("TD")[n];
                    y = rows[i + 1].getElementsByTagName("TD")[n];
                    if (dir == "asc") {
                        if (isNaN(x.innerHTML)) {
                            if (x.innerHTML.toLowerCase() > y.innerHTML.toLowerCase()) {
                                shouldSwitch = true;
                                break;
                            }
                        } else {
                            if (Number(x.innerHTML) > Number(y.innerHTML)) {
                                shouldSwitch = true;
                                break;
                            }
                        }
                    } else if (dir == "desc") {
                        if (isNaN(x.innerHTML)) {
                            if (x.innerHTML.toLowerCase() < y.innerHTML.toLowerCase()) {
                                shouldSwitch = true;
                                break;
                            }
                        } else {
                            if (Number(x.innerHTML) < Number(y.innerHTML)) {
                                shouldSwitch = true;
                                break;
                            }
                        }
                    }
                }
                if (shouldSwitch) {
                    rows[i].parentNode.insertBefore(rows[i + 1], rows[i]);
                    switching = true;
                    switchcount++;
                } else {
                    if (switchcount == 0 && dir == "asc") {
                        dir = "desc";
                        switching = true;
                    }
                }
            }
        }
    
        Object.keys(urls).forEach(key => {
            fetch(urls[key])
                .then(response => response.json())
                .then(data => {
                    let content = '';
                    if (Array.isArray(data)) {
                        content = createTable(data, `table-${key}`);
                    } else if (typeof data === 'object') {
                        let tableCounter = 0;
                        for (let dataKey in data) {
                            content += `<h2>${dataKey}</h2>`;
                            content += createTable(data[dataKey], `table-${key}-${tableCounter}`);
                            tableCounter++;
                        }
                    }
                    document.getElementById(key).innerHTML = content;
                    MathJax.Hub.Queue(["Typeset", MathJax.Hub])
                })
                .catch(error => {
                    console.error('Error:', error);
                    document.getElementById(key).innerHTML = `<p>Error loading data: ${error.message}</p>`;
                });
        });
    </script>
</body>
</html>