app.py

# Gradio Implementation: Lenix Carter
# License: BSD 3-Clause or CC-0

import warnings

import gradio as gr
import numpy as np
import matplotlib
import matplotlib.pyplot as plt

from sklearn.neural_network import MLPClassifier
from sklearn.preprocessing import MinMaxScaler
from sklearn import datasets
from sklearn.exceptions import ConvergenceWarning

plt.switch_backend('agg')

# different learning rate schedules and momentum parameters
params = [
    {
        "solver": "sgd",
        "learning_rate": "constant",
        "momentum": 0,
        "learning_rate_init": 0.2,
    },
    {
        "solver": "sgd",
        "learning_rate": "constant",
        "momentum": 0.9,
        "nesterovs_momentum": False,
        "learning_rate_init": 0.2,
    },
    {
        "solver": "sgd",
        "learning_rate": "constant",
        "momentum": 0.9,
        "nesterovs_momentum": True,
        "learning_rate_init": 0.2,
    },
    {
        "solver": "sgd",
        "learning_rate": "invscaling",
        "momentum": 0,
        "learning_rate_init": 0.2,
    },
    {
        "solver": "sgd",
        "learning_rate": "invscaling",
        "momentum": 0.9,
        "nesterovs_momentum": True,
        "learning_rate_init": 0.2,
    },
    {
        "solver": "sgd",
        "learning_rate": "invscaling",
        "momentum": 0.9,
        "nesterovs_momentum": False,
        "learning_rate_init": 0.2,
    },
    {"solver": "adam", "learning_rate_init": 0.01},
]

labels = [
    "constant learning-rate",
    "constant with momentum",
    "constant with Nesterov's momentum",
    "inv-scaling learning-rate",
    "inv-scaling with momentum",
    "inv-scaling with Nesterov's momentum",
    "adam",
]

plot_args = [
    {"c": "red", "linestyle": "-"},
    {"c": "green", "linestyle": "-"},
    {"c": "blue", "linestyle": "-"},
    {"c": "red", "linestyle": "--"},
    {"c": "green", "linestyle": "--"},
    {"c": "blue", "linestyle": "--"},
    {"c": "black", "linestyle": "-"},
]

# load / generate some toy datasets
iris = datasets.load_iris()
X_digits, y_digits = datasets.load_digits(return_X_y=True)
data_sets = [
    (iris.data, iris.target),
    (X_digits, y_digits),
    datasets.make_circles(noise=0.2, factor=0.5, random_state=1),
    datasets.make_moons(noise=0.3, random_state=0),
]

def run_mlp(dataset, models, clr_lr,
                             cwm_lr, cwm_mom,
                             nest_lr, nest_mom,
                             inv_lr,
                             iwm_lr, iwm_mom,
                             invN_lr, invN_mom,
                             adam_lr):
    plt.clf()       
    new_params = [
            {"learning_rate_init": clr_lr},
            {"learning_rate_init": cwm_lr,
             "momentum": cwm_mom},
            {"learning_rate_init": nest_lr,
             "momentum": nest_mom},
            {"learning_rate_init": inv_lr},
            {"learning_rate_init": iwm_lr,
             "momentum": iwm_mom},
            {"learning_rate_init": invN_lr,
             "momentum": invN_mom},
            {"learning_rate_init": adam_lr}
    ]
    for (param, new_param) in zip(params, new_params):
        param.update(new_param)

    iris = datasets.load_iris()
    X_digits, y_digits = datasets.load_digits(return_X_y=True)
    data_sets = [
        (iris.data, iris.target),
        (X_digits, y_digits),
        datasets.make_circles(noise=0.2, factor=0.5, random_state=1),
        datasets.make_moons(noise=0.3, random_state=0),
    ]
    name = ["Iris", "Digits", "Circles", "Moons"]

    return plot_on_dataset(*data_sets[dataset], models, name[dataset])

def plot_on_dataset(X, y, models, name):
    # for each dataset, plot learning for each learning strategy
    print("\nlearning on dataset %s" % name)

    X = MinMaxScaler().fit_transform(X)
    mlps = []
    if name == "Digits":
        # digits is larger but converges fairly quickly
        max_iter = 15
    else:
        max_iter = 400

    for model in models:
        label = labels[model]
        param = params[model]
        print("training: %s" % label)
        mlp = MLPClassifier(random_state=0, max_iter=max_iter, **param)

        # some parameter combinations will not converge as can be seen on the
        # plots so they are ignored here
        with warnings.catch_warnings():
            warnings.filterwarnings(
                "ignore", category=ConvergenceWarning, module="sklearn"
            )
            mlp.fit(X, y)

        mlps.append(mlp)
        print("Training set score: %f" % mlp.score(X, y))
        print("Training set loss: %f" % mlp.loss_)

        print(label)
        plt.plot(mlp.loss_curve_, label=label, **plot_args[model])

    plt.legend(loc="upper right")
    
    return plt

title = "Compare Stochastic learning strategies for MLPClassifier"
with gr.Blocks() as demo:
    gr.Markdown(f" # {title}")
    gr.Markdown("""
                This example demonstrates different stochastic learning strategies on the MLP Classifier. You may also tweak some parameters of the learning strategies.

                This is based on the example [here](https://scikit-learn.org/stable/auto_examples/neural_networks/plot_mlp_training_curves.html#sphx-glr-auto-examples-neural-networks-plot-mlp-training-curves-py)
                """)
    with gr.Tabs():
        with gr.TabItem("Model and Data Selection"):
            with gr.Row():
                dataset = gr.Dropdown(["Iris", "Digits", "Circles", "Moons"],
                                      value="Iris",
                                      type="index")
                models = gr.CheckboxGroup(["Constant Learning-Rate",
                                           "Constant with Momentum",
                                           "Constant with Nesterov's Momentum",
                                           "Inverse Scaling Learning-Rate",
                                           "Inverse Scaling with Momentum",
                                           "Inverse Scaling with Nesterov's Momentum",
                                           "Adam"],
                                          label="Stochastic Learning Strategy",
                                          type="index")
        with gr.TabItem("Model Tuning"):
            with gr.Accordion("Constant Learning-Rate", open=False):
                clr_lr = gr.Slider(0.01, 1.00, .2, label="Learning Rate")
            with gr.Accordion("Constant with Momentum", open=False):
                cwm_lr = gr.Slider(0.01, 1.00, .2, label="Learning Rate")
                cwm_mom = gr.Slider(0.01, 1.00, 0.9, label="Momentum")
            with gr.Accordion("Constant with Nesterov's Momentum", open=False):
                nest_lr = gr.Slider(0.01, 1.00, .2, label="Learning Rate")
                nest_mom = gr.Slider(0.01, 1.00, 0.9, label="Momentum")
            with gr.Accordion("Inverse Scaling Learning-Rate", open=False):
                inv_lr = gr.Slider(0.01, 1.00, .2, label="Learning Rate")
            with gr.Accordion("Inverse Scaling with Momentum", open=False):
                iwm_lr = gr.Slider(0.01, 1.00, .2, label="Learning Rate")
                iwm_mom = gr.Slider(0.01, 1.00, 0.9, label="Momentum")
            with gr.Accordion("Inverse Scaling with Nesterov's Momentum", open=False):
                invN_lr = gr.Slider(0.01, 1.00, .2, label="Learning Rate")
                invN_mom = gr.Slider(0.01, 1.00, 0.9, label="Momentum")
            with gr.Accordion("Adam", open=False):
                adam_lr = gr.Slider(0.001, 1.00, 0.01, label="Learning Rate")

    btn = gr.Button(label="Run")
    stoch_graph = gr.Plot(label="Stochastic Learning Strategies")
    btn.click(
            fn=run_mlp,
            inputs=[dataset, models,
                    clr_lr,
                    cwm_lr,
                    cwm_mom,
                    nest_lr,
                    nest_mom,
                    inv_lr,
                    iwm_lr,
                    iwm_mom,
                    invN_lr,
                    invN_mom,
                    adam_lr],
            outputs=[stoch_graph]
            )

if __name__ == '__main__':
    demo.launch()