eval demo w/ shuffling

app.py

@@ -8,89 +8,76 @@ import pandas as pd
 import torch
 import gradio as gr
 import openml
+from sklearn.model_selection import cross_val_score
 
 
-def compute(table: np.array):
-    vfunc = np.vectorize(lambda s: len(s))
-    non_empty_row_mask = (vfunc(table).sum(1) != 0)
-    table = table[non_empty_row_mask]
-    empty_mask = table == ''
-    empty_inds = np.where(empty_mask)
-    if not len(empty_inds[0]):
-        return "**Please leave at least one field blank for prediction.**", None
-    if not np.all(empty_inds[1][0] == empty_inds[1]):
-        return "**Please only leave fields of one column blank for prediction.**", None
-    y_column = empty_inds[1][0]
-    eval_lines = empty_inds[0]
-
-    train_table = np.delete(table, eval_lines, axis=0)
-    eval_table = table[eval_lines]
-
-    try:
-        x_train = torch.tensor(np.delete(train_table, y_column, axis=1).astype(np.float32))
-        x_eval = torch.tensor(np.delete(eval_table, y_column, axis=1).astype(np.float32))
-
-        y_train = train_table[:, y_column]
-    except ValueError:
-        return "**Please only add numbers (to the inputs) or leave fields empty.**", None
+def compute(file, y_attribute, cv_folds):
+    if file is None:
+        return 'Please upload a .arff file', y_attribute
+    if file.name.endswith('.arff'):
+        dataset = openml.datasets.OpenMLDataset('t', 'test', data_file=file.name)
+        X_, _, categorical_indicator_, attribute_names_ = dataset.get_data(
+            dataset_format="array")
+        if y_attribute not in attribute_names_:
+            return f"**Select attribute from {', '.join(attribute_names_)}**"
+        X, y, categorical_indicator_, attribute_names_ = dataset.get_data(
+            dataset_format="array", target=y_attribute)
+    else:
+        return 'Please upload a .arff file', y_attribute
+
+    order = np.arange(y.shape[0])
+    np.random.seed(13)
+    np.random.shuffle(order)
+    X, y = torch.tensor(X[order]), torch.tensor(y[order])
 
     classifier = TabPFNClassifier(base_path=tabpfn_path, device='cpu')
-    classifier.fit(x_train, y_train)
-    y_eval, p_eval = classifier.predict(x_eval, return_winning_probability=True)
+
+    scores = cross_val_score(classifier, X, y, cv=cv_folds, scoring='roc_auc_ovo')
+    print(scores)
+    # classifier.fit(x_train, y_train)
+    # y_eval, p_eval = classifier.predict(x_eval, return_winning_probability=True)
 
     # print(file, type(file))
-    out_table = table.copy().astype(str)
-    out_table[eval_lines, y_column] = [f"{y_e} (p={p_e:.2f})" for y_e, p_e in zip(y_eval, p_eval)]
-    return None, out_table
+    return f"ROC AUC OVO Cross Val mean is {sum(scores) / len(scores)} from {scores}. " + (
+        "The PFN is only trained for datasets with up to 1024 training examples and it had to extrapolate to greater datasets for this evaluation." if len(
+            y) // cv_folds > 1024 else ""), y_attribute
 
 
 def upload_file(file):
+    if file is None:
+        return
     if file.name.endswith('.arff'):
         dataset = openml.datasets.OpenMLDataset('t', 'test', data_file=file.name)
+        print(y_attribute)
         X_, _, categorical_indicator_, attribute_names_ = dataset.get_data(
-            dataset_format="array"
-        )
-        df = pd.DataFrame(X_, columns=attribute_names_)
-        return df
-    elif file.name.endswith('.csv') or file.name.endswith('.data'):
-        df = pd.read_csv(file.name, header=None)
-        df.columns = np.arange(len(df.columns))
-        print(df)
-        return df
+            dataset_format="array")
+        return f"Select attribute from {', '.join(attribute_names_)}", attribute_names_[-1]
+    else:
+        return 'Please upload a .arff file', None
 
 
-example = \
-    [
-        [1, 2, 1],
-        [2, 1, 1],
-        [1, 1, 1],
-        [2, 2, 2],
-        [3, 4, 2],
-        [3, 2, 2],
-        [2, 3, '']
-    ]
-
 with gr.Blocks() as demo:
     gr.Markdown("""This demo allows you to play with the **TabPFN**.
     You can either change the table manually (we have filled it with a toy benchmark, sum up to 3 has label 1 and over that label 2).
     The network predicts fields you leave empty. Only one column can have empty entries that are predicted.
     Please, provide everything but the label column as numeric values. It is ok to encode classes as integers.
     """)
-    inp_table = gr.DataFrame(type='numpy', value=example, headers=[''] * 3)
     inp_file = gr.File(
         label='Drop either a .csv (without header, only numeric values for all but the labels) or a .arff file.')
+    cv_folds = gr.Dropdown([2, 3, 4, 5], value=2, label='Number of CV folds')
+    out_text = gr.Markdown()
+
+    y_attribute = gr.Textbox(label='y attribute')
+
     examples = gr.Examples(examples=['iris.csv', 'balance-scale.arff'],
                            inputs=[inp_file],
-                           outputs=[inp_table],
+                           outputs=[out_text, y_attribute],
                            fn=upload_file,
                            cache_examples=True)
     btn = gr.Button("Predict Empty Table Cells")
+    # out_table = gr.DataFrame()
+    inp_file.change(fn=upload_file, inputs=inp_file, outputs=[out_text, y_attribute])
 
-    inp_file.change(fn=upload_file, inputs=inp_file, outputs=inp_table)
-
-    out_text = gr.Markdown()
-    out_table = gr.DataFrame()
-
-    btn.click(fn=compute, inputs=inp_table, outputs=[out_text, out_table])
+    btn.click(fn=compute, inputs=[inp_file, y_attribute, cv_folds], outputs=[out_text, y_attribute])
 
 demo.launch()