In [22]:

    import warnings
    warnings.filterwarnings("ignore")
    from IPython.core.display import display, HTML
    display(HTML("<style>.container { width:100% !important; }</style>"))

    import sys
    sys.path.append('..\\..\\pyclf\\models')
    sys.path.append('..\\..\\pyclf\\model')
    sys.path.append('..\\..\\pyclf\\validation')
    sys.path.append('..\\..\\pyclf\\pyutils')
    sys.path.append('..\\..\\bokeh')

    import os.path
    import json, copy, time, pickle
    import numpy as np
    import pandas as pd

    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.model_selection import GridSearchCV, StratifiedKFold
    from sklearn.linear_model import LogisticRegression

    import matplotlib.pyplot as plt
    import matplotlib as mpl
    %matplotlib inline
    %config InlineBackend.figure_format='retina'
    import seaborn as sns

    import experimental, validation, weight, multiclass_metrics, optimization
    from experimental import KNNWeightedFeatures
    from local_model import LocalModel

    import bokeh_confmat_v2
    from bokeh.io import output_notebook, show
    output_notebook()

    dataset = "..//..//training_data/trainset_v3.csv"

    #load configs
    with open('..//..//pyclf//configs/feature_headers.json', 'r') as f:
        feature_headers = json.load(f)

    with open('..//..//pyclf//configs//folds.json', 'r') as f:
        folds = json.load(f)

    with open('..//..//pyclf//configs//class_mapping.json', 'r') as f:
        mapper = json.load(f)

    mapper = mapper["loop_table"]

    #prepare data
    df = pd.read_csv(dataset, sep=';', decimal=',')
    df = df[df.class_name.isin(mapper.keys())]
    df.loc[:,'class_name'] = df.loc[:,'class_name'].map( lambda x: int(mapper[x]) if x in mapper.keys() else x)

    df[feature_headers["feature_names"]] = df[feature_headers["feature_names"]].applymap(lambda x: x.replace(',', '.'))

    classes = df.class_name.unique()
    X = df[feature_headers['feature_names']].as_matrix()
    y = df.class_name.as_matrix()
    y = np.array(y, dtype=np.int)

Loading BokehJS ...
In [5]:

    sns.set(font_scale=0.7)

    np.set_printoptions(precision=2, linewidth=300)
    optimizers = ["hC1", "hC2", "hC3", "hC4", "hC5", "fcHc1", "fcHc2", "fcHc3", "fcHc4", "cD1", "CDD"]

    fig, axs = plt.subplots(len(optimizers), 2 , sharex = False, figsize = (15, len(optimizers)*3))

    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        xopt = np.array(optimizerLog["_xopt"])
        fopt = optimizerLog["_fopt"]
        xopt = (xopt - xopt.min()) / (xopt.max() - xopt.min())

        title = optimizerLog["object"]
        if title == "HillClimb":
            title += " n_candiates:" + str(optimizerLog["n_candidates"])
        elif title == "FirstChoiceHillClimb":
            title += " mutation_k:" + str(optimizerLog["mutation_k"])
        axs[idx][0].set_title(title)
        axs[idx][0].plot(optimizerLog["_n_evals"], optimizerLog["fs_"])
        axs[idx][0].set_ylim([-0.930, -0.905])

        axs[idx][1].bar(np.arange(len(xopt)), xopt)
        axs[idx][1].set_title("Best weights")

In [33]:

    from sklearn.cross_validation import cross_val_score, StratifiedKFold

    NUMBER_OF_CV_RUNS = 50
    LOAD_PICKLED = True

    if not LOAD_PICKLED:
        optimizers = ["hC1", "hC2", "hC3", "hC4", "hC5", "fcHc1", "fcHc2", "fcHc3", "fcHc4", "cD1", "CDD"]

        scores = np.zeros((len(optimizers), NUMBER_OF_CV_RUNS))

        for i, optimizer in enumerate(optimizers):
            optimizerLog = json.load(open(optimizer + ".json"))
            knn = experimental.KNNWeightedFeatures(weights=optimizerLog["_xopt"])
            print optimizers[i]
            for j in range(NUMBER_OF_CV_RUNS):
                print j,
                scores[i][j] = cross_val_score(knn, X, y, scoring="f1_macro", cv=StratifiedKFold(y, 5, True )).mean()
            print " "

        scores_uniform = np.zeros((NUMBER_OF_CV_RUNS,))

        knn = experimental.KNNWeightedFeatures(weights=[1.0]*27)
        for i in range(NUMBER_OF_CV_RUNS):
            scores_uniform[i] = cross_val_score(knn, X, y, scoring="f1_macro", cv=StratifiedKFold(y, 5, True )).mean()

        scores_manual = np.zeros((NUMBER_OF_CV_RUNS,))

        knn = experimental.KNNWeightedFeatures(weights=[5.0] + [1.0]*26)
        for i in range(NUMBER_OF_CV_RUNS):
            scores_manual[i] = cross_val_score(knn, X, y, scoring="f1_macro", cv=StratifiedKFold(y, 5, True )).mean()

        scores = np.concatenate((scores_uniform.reshape(-1,1), scores.T), axis=1)
        scores_dump = dict()
        scores_dump["optimizers"] =  ["uniform weights"] + optimizers
        scores_dump["scores"] = scores
        pickle.dump(scores_dump, open("scores.dump", "wb"))

    if LOAD_PICKLED:
        f = open("scores.dump", "rb")
        scores_dump = pickle.load(f)
        f.close()
        scores = scores_dump["scores"]
        optimizers = scores_dump["optimizers"]

    sns.reset_orig()
    plt.figure(figsize=(9,4))
    sns.boxplot(data=scores,  saturation=0.7, showmeans=True, color='gray')
    plt.ylabel("$ \mathrm{F1_{macro}}$", fontsize=13)
    plt.xticks(np.arange(1 + len(optimizers)), optimizers, rotation=90);
    text = "hC1 - Modified Hill Climbing, 2 candidates\n"
    text += "hC2 - Modified Hill Climbing, 5 candidates\n"
    text += "hC3 - Modified Hill Climbing, 10 candidates\n"
    text += "hC4 - Modified Hill Climbing, 20 candidates\n"
    text += "hC5 - Modified Hill Climbing, 27 candidates\n"
    text += "fcHc1 - First-Choice Hill Climbing, mutation coef. 0.1 \n"
    text += "fcHc2 - First-Choice Hill Climbing, mutation coef. 0.2 \n"
    text += "fcHc3 - First-Choice Hill Climbing, mutation coef. 0.1, with simulated annealing \n"
    text += "fcHc4 - First-Choice Hill Climbing, mutation coef. 0.2, with simulated annealing \n"
    text += "cD - Coordinate Descent (line search) \n"
    text +=  "cDD - Coordinate Descent (dichotomous search)"
    plt.text(1, 0.901, text)
    plt.xlabel("Feature weighting method")
    plt.title("Comparison of different feature weighting methods")
    plt.grid()

    plt.savefig("optimizers_comparison.pdf", bbox_inches='tight', pad_inches=0.05)
    # plt.show()

In [35]:

    # %matplotlib

    plt.rcParams["font.family"] = "Roman"
    sns.set(font_scale=1.0)
    sns.set_color_codes()
    import matplotlib as mpl
    mpl.rcParams['text.usetex'] = True
    mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amsmath}']

    import matplotlib.gridspec as gridspec

    np.set_printoptions(precision=2, linewidth=300)
    optimizers = ["hC1", "hC2", "hC3", "hC4", "hC5", "fcHc1", "fcHc2", "fcHc3", "fcHc4", "cD1", "CDD"]

    colors = ['b', 'g', 'y']
    width = 3.5
    # plt.figure(figsize=(13, 5))

    plt.figure(figsize=(10, 5))
    G = gridspec.GridSpec(100, 100)

    plt.subplot(G[:40, :])
    for idx, optimizer in enumerate(["hC2", "hC3", "hC5"]):
        optimizerLog = json.load(open(optimizer + ".json"))
        xopt = np.array(optimizerLog["_xopt"])
        fopt = optimizerLog["_fopt"]
        xopt = (xopt - xopt.min()) / (xopt.max() - xopt.min())
        plt.bar(np.arange(len(xopt))*width + idx + 0.3, xopt, color=colors[idx], label = "number of candidates: " + str(optimizerLog["n_candidates"]))
        plt.legend()

    for idx in range(len(xopt)):
        plt.fill_between(np.linspace(idx*width, (idx+1)*width, 10), y1=1, alpha=0.1, color='b' if idx % 2 == 0 else 'y')

    plt.xticks(np.arange(len(xopt))*width + width/2, feature_headers["feature_names"], rotation=90)
    plt.ylim([-0.00, 1.0])
    plt.xlim([0.0, len(xopt)*width])
    plt.ylabel("Feature weight")

    plt.subplot(G[55:, :55])
    for idx, optimizer in enumerate(["hC2", "hC3", "hC5"]):
        optimizerLog = json.load(open(optimizer + ".json"))
        plt.plot(optimizerLog["fs_"], color=colors[idx],  label = "number of candidates: " + str(optimizerLog["n_candidates"]))

    plt.xlabel("iteration")
    plt.ylabel("$ \mathrm{-F1_{macro}}$", fontsize=13)
    plt.legend()

    plt.subplot(G[55:, 65:])
    for idx, optimizer in enumerate(["hC2", "hC3", "hC5"]):
        optimizerLog = json.load(open(optimizer + ".json"))
        plt.plot(optimizerLog["_n_evals"], color=colors[idx], label = "number of candidates: " + str(optimizerLog["n_candidates"]))

    plt.xlabel("iterations")
    plt.ylabel("number of function evaluations")
    plt.legend()
    # plt.show()
    plt.savefig("hC_weights.pdf")

In [36]:

    plt.rcParams["font.family"] = "Roman"
    sns.set(font_scale=1.0)
    sns.set_color_codes()
    import matplotlib as mpl
    mpl.rcParams['text.usetex'] = True
    mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amsmath}']

    import matplotlib.gridspec as gridspec

    np.set_printoptions(precision=2, linewidth=300)
    optimizers = ["fcHc1", "fcHc3" ]

    colors = ['b', 'g', 'y']
    width = 2.5
    # plt.figure(figsize=(13, 5))

    plt.figure(figsize=(10, 5))
    G = gridspec.GridSpec(100, 100)

    plt.subplot(G[:40, :])
    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        xopt = np.array(optimizerLog["_xopt"])
        fopt = optimizerLog["_fopt"]
        xopt = (xopt - xopt.min()) / (xopt.max() - xopt.min())
        plt.bar(np.arange(len(xopt))*width + idx + 0.3, xopt, color=colors[idx])

    plt.legend(["Without simulated annealing", "With simulated annealing"])

    for idx in range(len(xopt)):
        plt.fill_between(np.linspace(idx*width, (idx+1)*width, 10), y1=1, alpha=0.1, color='b' if idx % 2 == 0 else 'y')

    plt.xticks(np.arange(len(xopt))*width + width/2, feature_headers["feature_names"], rotation=90)
    plt.ylim([-0.00, 1.0])
    plt.xlim([0.0, len(xopt)*width])
    plt.ylabel("Feature weight")

    plt.subplot(G[55:, :55])
    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        plt.plot(optimizerLog["fs_"], color=colors[idx])

    plt.xlabel("iteration")
    plt.ylabel("$ \mathrm{-F1_{macro}}$", fontsize=13)
    plt.legend(["Without simulated annealing", "With simulated annealing"])

    plt.subplot(G[55:, 65:])
    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        plt.plot(optimizerLog["_n_evals"], color=colors[idx])

    plt.xlabel("iterations")
    plt.ylabel("number of function evaluations")
    plt.legend(["Without simulated annealing", "With simulated annealing"])
    # plt.show()
    plt.savefig("fcHc_weights.pdf")

In [37]:

    plt.rcParams["font.family"] = "Roman"
    sns.set(font_scale=1.0)
    sns.set_color_codes()
    import matplotlib as mpl
    mpl.rcParams['text.usetex'] = True
    mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amsmath}']

    import matplotlib.gridspec as gridspec

    np.set_printoptions(precision=2, linewidth=300)
    optimizers = [ "cD1", "CDD" ]

    colors = ['b', 'g', 'y']
    width = 2.5
    # plt.figure(figsize=(13, 5))

    plt.figure(figsize=(10, 5))
    G = gridspec.GridSpec(100, 100)

    plt.subplot(G[:40, :])
    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        xopt = np.array(optimizerLog["_xopt"])
        fopt = optimizerLog["_fopt"]
        xopt = (xopt - xopt.min()) / (xopt.max() - xopt.min())
        plt.bar(np.arange(len(xopt))*width + idx + 0.3, xopt, color=colors[idx])

    plt.legend(["Line Search", "Binary search"])

    for idx in range(len(xopt)):
        plt.fill_between(np.linspace(idx*width, (idx+1)*width, 10), y1=1, alpha=0.1, color='b' if idx % 2 == 0 else 'y')

    plt.xticks(np.arange(len(xopt))*width + width/2, feature_headers["feature_names"], rotation=90)
    plt.ylim([-0.00, 1.0])
    plt.xlim([0.0, len(xopt)*width])
    plt.ylabel("Feature weight")

    plt.subplot(G[55:, :55])
    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        plt.plot(optimizerLog["fs_"], color=colors[idx])

    plt.xlabel("iteration")
    plt.ylabel("$ \mathrm{-F1_{macro}}$", fontsize=13)
    plt.legend(["Line Search", "Binary search"])

    plt.subplot(G[55:, 65:])
    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        plt.plot(optimizerLog["_n_evals"], color=colors[idx])

    plt.xlabel("iterations")
    plt.ylabel("number of function evaluations")
    plt.legend(["Line Search", "Binary search"])
    plt.savefig("cD_weights.pdf")

In [19]:

    plt.plot([1,2,3,4,5])
    plt.show()

    Using matplotlib backend: Qt4Agg

In [ ]: