In [42]:

    import warnings
    warnings.filterwarnings("ignore")
    from IPython.core.display import display, HTML
    display(HTML("<style>.container { width:100% !important; }</style>"))

    import sys
    sys.path.append('..\\..\\pyclf\\models')
    sys.path.append('..\\..\\pyclf\\model')
    sys.path.append('..\\..\\pyclf\\validation')
    sys.path.append('..\\..\\pyclf\\pyutils')
    sys.path.append('..\\..\\bokeh')

    import os.path
    import json, copy, time, pickle
    import numpy as np
    import pandas as pd

    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import MinMaxScaler, StandardScaler, PolynomialFeatures
    from sklearn.feature_selection import SelectFromModel
    from sklearn.model_selection import GridSearchCV, StratifiedKFold
    from sklearn.linear_model import LogisticRegression

    import matplotlib.pyplot as plt
    import matplotlib as mpl
    %matplotlib inline
    %config InlineBackend.figure_format='retina'
    import seaborn as sns

    import experimental, validation, weight, multiclass_metrics, optimization
    from experimental import KNNWeightedFeatures
    from local_model import LocalModel

    import bokeh_confmat_v2
    from bokeh.io import output_notebook, show
    output_notebook()

    dataset = "..//..//training_data/trainset_v3.csv"

    #load configs
    with open('..//..//pyclf//configs/feature_headers.json', 'r') as f:
        feature_headers = json.load(f)

    with open('..//..//pyclf//configs//folds.json', 'r') as f:
        folds = json.load(f)

    with open('..//..//pyclf//configs//class_mapping.json', 'r') as f:
        mapper = json.load(f)

    mapper = mapper["loop_table"]

    #prepare data
    df = pd.read_csv(dataset, sep=';', decimal=',')
    df = df[df.class_name.isin(mapper.keys())]
    df.loc[:,'class_name'] = df.loc[:,'class_name'].map( lambda x: int(mapper[x]) if x in mapper.keys() else x)

    df[feature_headers["feature_names"]] = df[feature_headers["feature_names"]].applymap(lambda x: x.replace(',', '.'))

    classes = df.class_name.unique()
    X = df[feature_headers['feature_names']].as_matrix()
    y = df.class_name.as_matrix()
    y = np.array(y, dtype=np.int)

    class NamedFunction:
        def __init__(self, name, f):
            self.f = f
            self.name = name

        def __call__(self, *args, **kwargs):
            return self.f(*args, **kwargs)

        def __str__(self):
            return self.name

        def __repr__(self):
            return self.name

    folds_ind = [ (f["train_ind"], f["test_ind"]) for f in folds["folds"]]

    dist_tf1 = NamedFunction('1.0  / x', lambda x: 1.0 / x)
    dist_tf2 = NamedFunction('exp(-x^2) / 0.5', lambda x: np.exp(-x*x) / 0.5)
    dist_tf3 = NamedFunction('exp(-x^2) / 1.0', lambda x: np.exp(-x*x) / 1.0)
    dist_tf4 = NamedFunction('exp(-x^2) / 1.0', lambda x: np.exp(-x*x) / 1.5)
    dist_tf5 = NamedFunction('exp(-x^2) / 2.0', lambda x: np.exp(-x*x) / 2.0)
    dist_tf6 = NamedFunction('exp(-x^2) / 3.0', lambda x: np.exp(-x*x) / 3.0)
    dist_tf7 = NamedFunction('exp(-x^2) / 4.0', lambda x: np.exp(-x*x) / 4.0)
    dist_tf8 = NamedFunction('exp(-x^2) / 4.0', lambda x: np.exp(-x*x) / 5.0)

    cd_weights = np.array([3.1, 0.9,  1.0,  1.0,  1.0,  1.1,  0.0,  0.5,  0.7,  1.0,  0.9,  1.0,  0.9,  1.0,  1.0,  1.0,  1.0,  0.9,  0.9,  1.0,  1.1,  1.2,  1.0,  1.0,  1.0, 1.0, 1.0])

Loading BokehJS ...
In [69]:

    sns.reset_orig()

    plt.rcParams["font.family"] = "Roman"
    # sns.set(font_scale=1.0)
    # sns.set_color_codes()
    import matplotlib as mpl
    mpl.rcParams['text.usetex'] = True
    mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amsmath}']

    local_logit_50CV = pickle.load(open("..//knn with local model//scores_50_CV","r"))
    logit_50CV = pickle.load(open("..//logit//logit_50CV_scores", "r"))
    knn_50CV = pickle.load(open("..//features weight optimization//scores.dump", "r"))
    logit_50CV = np.array(logit_50CV).mean(1)
    knn_50CV = knn_50CV["scores"]
    local_logit_50CV = np.array(local_logit_50CV['scores_50CV_local'])

    plt.figure(figsize=(7,3))
    scores = np.concatenate((knn_50CV[:, [0,4]], logit_50CV.reshape(-1,1), local_logit_50CV.reshape(-1,1)), axis=1)
    sns.boxplot(data=scores, showmeans=True, orient='horizontal')
    plt.yticks(range(4), ["Nearest Neighbor (NN) with uniform features", "NN with weighted features", "Logistic regression", "Locally weighted logistc regression"], fontsize=10)
    plt.xlabel("$ \mathrm{F1_{macro}}$");
    plt.ylabel("Classifier")
    plt.grid()
    plt.savefig("classifiers_comparasion.pdf",  bbox_inches='tight', pad_inches=0.05)

    # text = "NN - Nearest Neighbor"
    # text += "Logi"
    # plt.text(1, 0.905, text)

In [49]:

    #---------------------------------------------------------------------------

    knn = Pipeline([
    ("minmax", MinMaxScaler()),
    ("weights", weight.Weight(cd_weights)),
    ("knn", KNeighborsClassifier(n_neighbors=1)) ])

    #---------------------------------------------------------------------------

    X_scaled = StandardScaler().fit_transform(X)
    X_poly = PolynomialFeatures(include_bias=False, degree=2).fit_transform(X_scaled)
    logit =  LogisticRegression(C=1.0, penalty='l2', class_weight=None)
    logit.fit(X_poly, y)

    logit = Pipeline([("std_scaler", StandardScaler()),
                      ("poly", PolynomialFeatures(include_bias=False, degree=2)),
                      ("feature_selection", SelectFromModel(logit, threshold=3, prefit=False)),
                      ("logit", LogisticRegression(C=1.0))])

    #---------------------------------------------------------------------------

    local_logit = LocalModel(weights=cd_weights,
                             pca=None,
                             local_model=LogisticRegression(C=100.0),
                             n_neighbors=200,
                             dist_tf=lambda x: 1.0 / x)

    #---------------------------------------------------------------------------

In [50]:

    print "validating knn"
    val_knn = validation.Validation(knn, X, y, sorted(set(y)), n_folds='own', folds=folds["folds"])
    print val_knn.report
    print "\n\n"
    print "validating local logit"
    val_local_logit = validation.Validation(local_logit, X, y, sorted(set(y)), n_folds='own', folds=folds["folds"])
    print val_local_logit.report
    print "validating logit"
    val_logit = validation.Validation(logit, X, y, sorted(set(y)), n_folds='own', folds=folds["folds"])
    print val_logit.report
    print "\n\n"

    validating knn
                                         precision          recall           f1 score    support
                                 1   1.00 (+/-0.00)   0.98 (+/-0.07)   0.99 (+/-0.03)   118
                                 2   0.95 (+/-0.02)   0.96 (+/-0.02)   0.96 (+/-0.01)  1766
                                 3   0.94 (+/-0.03)   0.94 (+/-0.05)   0.94 (+/-0.02)   582
                                 4   0.87 (+/-0.02)   0.89 (+/-0.04)   0.88 (+/-0.02)  1084
                                 5   0.89 (+/-0.01)   0.89 (+/-0.02)   0.89 (+/-0.01)  2176
                                 6   0.77 (+/-0.09)   0.72 (+/-0.03)   0.74 (+/-0.05)   645
                                 7   0.95 (+/-0.03)   0.95 (+/-0.04)   0.95 (+/-0.01)  1001
                                 8   0.97 (+/-0.02)   0.98 (+/-0.01)   0.97 (+/-0.01)  2103
                                 9   0.98 (+/-0.03)   0.97 (+/-0.03)   0.97 (+/-0.02)   434

    f1_macro train:            1.000 (+/-0.000)
    f1_macro test:             0.922 (+/-0.005)
    overall test accuracy:    0.923

    validating local logit
                                         precision          recall           f1 score    support
                                 1   0.99 (+/-0.03)   0.99 (+/-0.03)   0.99 (+/-0.02)   118
                                 2   0.97 (+/-0.02)   0.97 (+/-0.02)   0.97 (+/-0.01)  1766
                                 3   0.94 (+/-0.03)   0.95 (+/-0.02)   0.95 (+/-0.02)   582
                                 4   0.92 (+/-0.02)   0.90 (+/-0.04)   0.91 (+/-0.02)  1084
                                 5   0.91 (+/-0.01)   0.94 (+/-0.01)   0.92 (+/-0.01)  2176
                                 6   0.82 (+/-0.06)   0.76 (+/-0.04)   0.79 (+/-0.04)   645
                                 7   0.96 (+/-0.02)   0.96 (+/-0.04)   0.96 (+/-0.02)  1001
                                 8   0.98 (+/-0.01)   0.98 (+/-0.00)   0.98 (+/-0.01)  2103
                                 9   0.98 (+/-0.03)   0.97 (+/-0.02)   0.98 (+/-0.02)   434

    f1_macro train:            0.599 (+/-0.028)
    f1_macro test:             0.938 (+/-0.006)
    overall test accuracy:    0.942
    validating logit
                                         precision          recall           f1 score    support
                                 1   0.96 (+/-0.05)   0.93 (+/-0.10)   0.94 (+/-0.05)   118
                                 2   0.95 (+/-0.01)   0.96 (+/-0.01)   0.96 (+/-0.01)  1766
                                 3   0.92 (+/-0.04)   0.94 (+/-0.04)   0.93 (+/-0.03)   582
                                 4   0.89 (+/-0.04)   0.89 (+/-0.03)   0.89 (+/-0.03)  1084
                                 5   0.89 (+/-0.01)   0.92 (+/-0.01)   0.90 (+/-0.01)  2176
                                 6   0.80 (+/-0.06)   0.70 (+/-0.06)   0.75 (+/-0.05)   645
                                 7   0.95 (+/-0.03)   0.95 (+/-0.05)   0.95 (+/-0.03)  1001
                                 8   0.98 (+/-0.02)   0.97 (+/-0.01)   0.97 (+/-0.01)  2103
                                 9   0.98 (+/-0.05)   0.97 (+/-0.01)   0.97 (+/-0.02)   434

    f1_macro train:            0.945 (+/-0.004)
    f1_macro test:             0.919 (+/-0.005)
    overall test accuracy:    0.927

In [51]:

    cross_validation_local_logit = dict(y_true = val_local_logit.y_true, y_pred = val_local_logit.y_pred)
    cross_validation_logit = dict(y_true = val_logit.y_true, y_pred = val_logit.y_pred)
    cross_validation_knn = dict(y_true = val_knn.y_true, y_pred = val_knn.y_pred)

In [52]:

    pickle.dump(cross_validation_local_logit, open("cross_validation_local_logit", "wb"))
    pickle.dump(cross_validation_logit, open("cross_validation_logit", "wb"))
    pickle.dump(cross_validation_knn, open("cross_validation_knn", "wb"))

In [53]:

    m_knn = multiclass_metrics.MulticlassMetrics(cross_validation_knn["y_true"], cross_validation_knn["y_pred"])
    m_local = multiclass_metrics.MulticlassMetrics(cross_validation_local_logit["y_true"], cross_validation_local_logit["y_pred"])
    m_logit = multiclass_metrics.MulticlassMetrics(cross_validation_logit["y_true"], cross_validation_logit["y_pred"])

In [55]:

    from sklearn import metrics
    print "accuracy: "      + str(metrics.accuracy_score(m_knn.y_true, m_knn.y_pred))
    print "f macro:  "      + str(m_knn.macro_fscore)
    print "f micro:  "      + str(m_knn.micro_fscore)
    print "recall macro:     " + str(m_knn.macro_recall)
    print "recall micro:     " + str(m_knn.micro_recall)
    print "precision macro:  " + str(m_knn.macro_precision)
    print "precision micro:  " + str(m_knn.micro_precision)

    accuracy: 0.923302048643
    f macro:  0.921796236574
    f micro:  0.923302048643
    recall macro:     0.919270012088
    recall micro:     0.923302048643
    precision macro:  0.924529356519
    precision micro:  0.923302048643

In [56]:

    print "accuracy: "      + str(metrics.accuracy_score(m_local.y_true, m_local.y_pred))
    print "f macro:  "      + str(m_local.macro_fscore)
    print "f micro:  "      + str(m_local.micro_fscore)
    print "recall macro:     " + str(m_local.macro_recall)
    print "recall micro:     " + str(m_local.micro_recall)
    print "precision macro:  " + str(m_local.macro_precision)
    print "precision micro:  " + str(m_local.micro_precision)

    accuracy: 0.941669189626
    f macro:  0.938354817275
    f micro:  0.941669189626
    recall macro:     0.935648019367
    recall micro:     0.941669189626
    precision macro:  0.941461723167
    precision micro:  0.941669189626

In [57]:

    print "accuracy: "      + str(metrics.accuracy_score(m_logit.y_true, m_logit.y_pred))
    print "f macro:  "      + str(m_logit.macro_fscore)
    print "f micro:  "      + str(m_logit.micro_fscore)
    print "recall macro:     " + str(m_logit.macro_recall)
    print "recall micro:     " + str(m_logit.micro_recall)
    print "precision macro:  " + str(m_logit.macro_precision)
    print "precision micro:  " + str(m_logit.micro_precision)

    accuracy: 0.926834191139
    f macro:  0.918865296173
    f micro:  0.926834191139
    recall macro:     0.914285029459
    recall micro:     0.926834191139
    precision macro:  0.924472911676
    precision micro:  0.926834191139

In [71]:

    sns.reset_orig()

    plt.rcParams["font.family"] = "Roman"
    # sns.set(font_scale=1.0)
    # sns.set_color_codes()
    import matplotlib as mpl
    mpl.rcParams['text.usetex'] = True
    mpl.rcParams['text.latex.preamble'] = [r'\usepackage{amsmath}']

    sns.set(font_scale=1.0)

    # plt.figure(figsize=(12,12))
    # plt.subplot(1,3,1)
    # m_knn.plot_confusion_matrix(title="Nearest Neighbor with Weighted Features", fontsize=10); plt.grid()
    # plt.subplot(1,3,2)
    # m_logit.plot_confusion_matrix(title="Logistic Regression", fontsize=10); plt.grid()
    # plt.subplot(1,3,3)
    # m_local.plot_confusion_matrix(title="Locally Weighted Logistic Regression", fontsize=10); plt.grid()

    # plt.figure(figsize=(12,12))
    # # plt.suptitle("Confusion normalized across columns")
    # plt.subplot(1,3,1)
    # m_knn.plot_confusion_matrix(0, title="Nearest Neighbor with Weighted Features", fontsize=10); plt.grid()
    # plt.subplot(1,3,2)
    # m_logit.plot_confusion_matrix(0, title="Logistic Regression", fontsize=10); plt.grid()
    # plt.subplot(1,3,3)
    # m_local.plot_confusion_matrix(0, title="Locally Weighted Logistic Regression", fontsize=10); plt.grid()

    plt.figure(figsize=(12,12))
    # plt.suptitle("Confusion normalized across rows")
    plt.subplot(1,3,1)
    m_knn.plot_confusion_matrix(1, title="Nearest Neighbor with Weighted Features", fontsize=10); plt.grid()
    plt.subplot(1,3,2)
    m_logit.plot_confusion_matrix(1, title="Logistic Regression", fontsize=10); plt.grid()
    plt.subplot(1,3,3)
    m_local.plot_confusion_matrix(1, title="Locally Weighted Logistic Regression", fontsize=10); plt.grid()
    plt.savefig("confusion_matrices_norm_1.pdf",  bbox_inches='tight', pad_inches=0.05)

In [54]:

    print "KNN"
    print val_knn.report
    print "\n\nLogistic regression"
    print val_logit.report
    print "\n\nLocal model"
    print val_local_logit.report

    KNN
                                         precision          recall           f1 score    support
                                 1   1.00 (+/-0.00)   0.98 (+/-0.07)   0.99 (+/-0.03)   118
                                 2   0.96 (+/-0.03)   0.96 (+/-0.02)   0.96 (+/-0.01)  1766
                                 3   0.95 (+/-0.04)   0.94 (+/-0.01)   0.94 (+/-0.03)   624
                                 4   0.86 (+/-0.05)   0.88 (+/-0.03)   0.87 (+/-0.03)  1142
                                 5   0.89 (+/-0.03)   0.91 (+/-0.03)   0.90 (+/-0.03)  2170
                                 6   0.79 (+/-0.06)   0.73 (+/-0.07)   0.76 (+/-0.05)   645
                                 7   0.97 (+/-0.03)   0.96 (+/-0.02)   0.96 (+/-0.01)  1072
                                 8   0.98 (+/-0.01)   0.99 (+/-0.01)   0.99 (+/-0.01)  1950
                                 9   0.98 (+/-0.02)   0.98 (+/-0.02)   0.98 (+/-0.02)   422

    f1_macro train:            1.000 (+/-0.000)
    f1_macro test:             0.928 (+/-0.012)
    overall test accuracy:    0.929

    Logistic regression
                                         precision          recall           f1 score    support
                                 1   0.97 (+/-0.08)   0.97 (+/-0.10)   0.97 (+/-0.04)   118
                                 2   0.96 (+/-0.01)   0.96 (+/-0.01)   0.96 (+/-0.01)  1766
                                 3   0.94 (+/-0.04)   0.93 (+/-0.03)   0.94 (+/-0.02)   624
                                 4   0.88 (+/-0.02)   0.90 (+/-0.02)   0.89 (+/-0.02)  1142
                                 5   0.91 (+/-0.02)   0.92 (+/-0.02)   0.92 (+/-0.02)  2170
                                 6   0.80 (+/-0.07)   0.76 (+/-0.03)   0.78 (+/-0.03)   645
                                 7   0.96 (+/-0.03)   0.94 (+/-0.03)   0.95 (+/-0.02)  1072
                                 8   0.98 (+/-0.01)   0.98 (+/-0.02)   0.98 (+/-0.01)  1950
                                 9   0.98 (+/-0.03)   0.98 (+/-0.02)   0.98 (+/-0.02)   422

    f1_macro train:            0.977 (+/-0.001)
    f1_macro test:             0.928 (+/-0.008)
    overall test accuracy:    0.933

    Local model
                                         precision          recall           f1 score    support
                                 1   1.00 (+/-0.00)   0.99 (+/-0.03)   1.00 (+/-0.02)   118
                                 2   0.97 (+/-0.01)   0.97 (+/-0.02)   0.97 (+/-0.01)  1766
                                 3   0.96 (+/-0.05)   0.96 (+/-0.02)   0.96 (+/-0.02)   624
                                 4   0.90 (+/-0.05)   0.88 (+/-0.03)   0.89 (+/-0.03)  1142
                                 5   0.90 (+/-0.01)   0.93 (+/-0.03)   0.91 (+/-0.02)  2170
                                 6   0.83 (+/-0.06)   0.75 (+/-0.06)   0.78 (+/-0.04)   645
                                 7   0.97 (+/-0.02)   0.97 (+/-0.02)   0.97 (+/-0.01)  1072
                                 8   0.98 (+/-0.01)   0.99 (+/-0.01)   0.99 (+/-0.01)  1950
                                 9   0.98 (+/-0.03)   0.99 (+/-0.02)   0.98 (+/-0.02)   422

    f1_macro train:            0.996 (+/-0.001)
    f1_macro test:             0.939 (+/-0.004)
    overall test accuracy:    0.940

In [9]:

    from sklearn.ensemble import GradientBoostingClassifier
    gbc = GradientBoostingClassifier(max_depth=4)
    val_gbc = validation.Validation(gbc, X, y, sorted(set(y)), n_folds='own', folds=folds["folds"])
    print val_gbc.report

                                         precision          recall           f1 score    support
                                 1   0.98 (+/-0.06)   0.98 (+/-0.07)   0.98 (+/-0.04)   118
                                 2   0.96 (+/-0.02)   0.97 (+/-0.01)   0.96 (+/-0.01)  1766
                                 3   0.94 (+/-0.04)   0.92 (+/-0.03)   0.93 (+/-0.03)   624
                                 4   0.92 (+/-0.03)   0.85 (+/-0.02)   0.88 (+/-0.03)  1142
                                 5   0.87 (+/-0.03)   0.94 (+/-0.02)   0.90 (+/-0.02)  2170
                                 6   0.79 (+/-0.02)   0.67 (+/-0.08)   0.72 (+/-0.06)   645
                                 7   0.96 (+/-0.02)   0.96 (+/-0.02)   0.96 (+/-0.02)  1072
                                 8   0.98 (+/-0.01)   0.99 (+/-0.01)   0.98 (+/-0.01)  1950
                                 9   0.99 (+/-0.01)   0.96 (+/-0.04)   0.98 (+/-0.03)   422

    f1_macro train:            0.994 (+/-0.001)
    f1_macro test:             0.922 (+/-0.012)
    overall test accuracy:    0.928

In [7]:

    cross_validation_local_logit = pickle.load(open("cross_validation_local_logit", "r"))
    m_local = multiclass_metrics.MulticlassMetrics(cross_validation_local_logit["y_true"], cross_validation_local_logit["y_pred"])

In [31]:

    %matplotlib inline

    # with implicit fonts "ďěňřťů" are not displayed properly in the plot title
    from matplotlib import rcParams
    rcParams['font.family'] = 'sans-serif'
    rcParams['text.latex.unicode'] = True

    if "Bitstream Vera Sans" in rcParams['font.sans-serif']:
        rcParams['font.sans-serif'].remove("Bitstream Vera Sans")

    sns.set(font_scale=1.0)
    plt.figure(figsize=(4,4))
    m_local.plot_confusion_matrix(fontsize=12); plt.grid()
    plt.title(u"Matice záměn, lokalně platný model", fontsize=16)

Out[31]:

    <matplotlib.text.Text at 0x1266f9e8>

In [4]:

    w = [1.0]*27
    knn = experimental.KNNWeightedFeatures(w)
    val_knn = validation.Validation(knn, X, y, sorted(set(y)), n_folds='own', folds=folds["folds"])

    cross_validation_knn = dict(y_true = val_knn.y_true, y_pred = val_knn.y_pred)

    m_knn = multiclass_metrics.MulticlassMetrics(cross_validation_knn["y_true"], cross_validation_knn["y_pred"])

    print "accuracy: "      + str(metrics.accuracy_score(m_knn.y_true, m_knn.y_pred))
    print "f macro:  "      + str(m_knn.macro_fscore)
    print "f micro:  "      + str(m_knn.micro_fscore)
    print "recall macro:     " + str(m_knn.macro_recall)
    print "recall micro:     " + str(m_knn.micro_recall)
    print "precision macro:  " + str(m_knn.macro_precision)
    print "precision micro:  " + str(m_knn.micro_precision)

    accuracy: 0.914017559794
    f macro:  0.907642260698
    f micro:  0.914017559794
    recall macro:     0.902650164175
    recall micro:     0.914017559794
    precision macro:  0.913204876401
    precision micro:  0.914017559794

In [ ]: