In [44]:

    import warnings
    warnings.filterwarnings("ignore")
    from IPython.core.display import display, HTML
    display(HTML("<style>.container { width:100% !important; }</style>"))

    import sys
    sys.path.append('..\\..\\pyclf\\models')
    sys.path.append('..\\..\\pyclf\\model')
    sys.path.append('..\\..\\pyclf\\validation')
    sys.path.append('..\\..\\pyclf\\pyutils')
    sys.path.append('..\\..\\bokeh')

    import os.path
    import json, copy, time, pickle
    import numpy as np
    import pandas as pd

    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.model_selection import GridSearchCV, StratifiedKFold
    from sklearn.linear_model import LogisticRegression

    import matplotlib.pyplot as plt
    import matplotlib as mpl
    %matplotlib inline
    %config InlineBackend.figure_format='retina'
    import seaborn as sns

    import experimental, validation, weight, multiclass_metrics, optimization
    from experimental import KNNWeightedFeatures
    from local_model import LocalModel

    import bokeh_confmat_v2
    from bokeh.io import output_notebook, show
    output_notebook()

    dataset = "..//..//training_data/trainset_v3.csv"

    #load configs
    with open('..//..//pyclf//configs/feature_headers.json', 'r') as f:
        feature_headers = json.load(f)

    with open('..//..//pyclf//configs//folds.json', 'r') as f:
        folds = json.load(f)

    with open('..//..//pyclf//configs//class_mapping.json', 'r') as f:
        mapper = json.load(f)

    mapper = mapper["loop_table"]

    #prepare data
    df = pd.read_csv(dataset, sep=';', decimal=',')
    df = df[df.class_name.isin(mapper.keys())]
    df.loc[:,'class_name'] = df.loc[:,'class_name'].map( lambda x: int(mapper[x]) if x in mapper.keys() else x)

    df[feature_headers["feature_names"]] = df[feature_headers["feature_names"]].applymap(lambda x: x.replace(',', '.'))

    classes = df.class_name.unique()
    X = df[feature_headers['feature_names']].as_matrix()
    y = df.class_name.as_matrix()
    y = np.array(y, dtype=np.int)

    class NamedFunction:
        def __init__(self, name, f):
            self.f = f
            self.name = name

        def __call__(self, *args, **kwargs):
            return self.f(*args, **kwargs)

        def __str__(self):
            return self.name

        def __repr__(self):
            return self.name

    folds_ind = [ (f["train_ind"], f["test_ind"]) for f in folds["folds"]]

    dist_tf1 = NamedFunction('1.0  / x', lambda x: 1.0 / x)
    dist_tf2 = NamedFunction('exp(-x^2) / 0.5', lambda x: np.exp(-x*x) / 0.5)
    dist_tf3 = NamedFunction('exp(-x^2) / 1.0', lambda x: np.exp(-x*x) / 1.0)
    dist_tf4 = NamedFunction('exp(-x^2) / 1.0', lambda x: np.exp(-x*x) / 1.5)
    dist_tf5 = NamedFunction('exp(-x^2) / 2.0', lambda x: np.exp(-x*x) / 2.0)
    dist_tf6 = NamedFunction('exp(-x^2) / 3.0', lambda x: np.exp(-x*x) / 3.0)
    dist_tf7 = NamedFunction('exp(-x^2) / 4.0', lambda x: np.exp(-x*x) / 4.0)
    dist_tf8 = NamedFunction('exp(-x^2) / 4.0', lambda x: np.exp(-x*x) / 5.0)

    cd_weights = np.array([3.1, 0.9,  1.0,  1.0,  1.0,  1.1,  0.0,  0.5,  0.7,  1.0,  0.9,  1.0,  0.9,  1.0,  1.0,  1.0,  1.0,  0.9,  0.9,  1.0,  1.1,  1.2,  1.0,  1.0,  1.0, 1.0, 1.0])

Loading BokehJS ...
In [45]:

    reload(experimental)
    warnings.filterwarnings("ignore")

    local_logit = LocalModel(weights=cd_weights,
                             pca=None,
                             local_model=LogisticRegression(C=1.0),
                             n_neighbors=1,
                             dist_tf=None)

    param_grid = {
                  'local_model__class_weight': [None],
                  "pca"                  : [None],
                  "local_model__C": [1.0, 10.0, 100.0, 1000.0, 10000.0, 10000.0],
                  "n_neighbors"          : [20, 50, 100, 150, 200, 300],
                  "dist_tf"       : [None, dist_tf1, dist_tf3]
                 }

    grid_search = GridSearchCV(local_logit,
                               param_grid=param_grid,
                               scoring='f1_macro',
                               cv = folds_ind,
                               return_train_score=False)
    t1 = time.time()
    grid_search.fit(X, y)
    t2 = time.time()
    print t2 - t1
    grid_search.best_score_, grid_search.best_params_

    7114.66599989

Out[45]:

    (0.93833015422009258,
     {'dist_tf': 1.0  / x,
      'local_model__C': 100.0,
      'local_model__class_weight': None,
      'n_neighbors': 200,
      'pca': None})

In [55]:

    cv_results = pd.DataFrame(grid_search.cv_results_)
    cv_results.to_csv('local_model_grid_seach.csv', sep=';')
    cv_results.sort('rank_test_score').head(5)

Out[55]:

| mean_fit_time | mean_score_time | mean_test_score | param_dist_tf | param_local_model__C | param_local_model__class_weight | param_n_neighbors | param_pca | params | rank_test_score | split0_test_score | split1_test_score | split2_test_score | split3_test_score | split4_test_score | std_fit_time | std_score_time | std_test_score
--- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---
52 | 0.1104 | 16.7034 | 0.938330 | 1.0 / x | 100 | None | 200 | None | {u'n_neighbors': 200, u'dist_tf': 1.0 / x, u'... | 1 | 0.936755 | 0.938030 | 0.934161 | 0.943624 | 0.939162 | 0.009113 | 0.860685 | 0.003114
51 | 0.1124 | 12.0086 | 0.936975 | 1.0 / x | 100 | None | 150 | None | {u'n_neighbors': 150, u'dist_tf': 1.0 / x, u'... | 2 | 0.933083 | 0.938765 | 0.930622 | 0.941746 | 0.940802 | 0.009851 | 0.390808 | 0.004381
53 | 0.1082 | 27.0706 | 0.936788 | 1.0 / x | 100 | None | 300 | None | {u'n_neighbors': 300, u'dist_tf': 1.0 / x, u'... | 3 | 0.933460 | 0.935253 | 0.933764 | 0.944162 | 0.937434 | 0.009621 | 0.504215 | 0.003928
50 | 0.1228 | 8.1938 | 0.936731 | 1.0 / x | 100 | None | 100 | None | {u'n_neighbors': 100, u'dist_tf': 1.0 / x, u'... | 4 | 0.935219 | 0.938253 | 0.932597 | 0.941268 | 0.936364 | 0.016869 | 0.361140 | 0.002908
45 | 0.1080 | 9.7632 | 0.935303 | 1.0 / x | 10 | None | 150 | None | {u'n_neighbors': 150, u'dist_tf': 1.0 / x, u'... | 5 | 0.926727 | 0.935210 | 0.933013 | 0.940504 | 0.941334 | 0.002608 | 0.325419 | 0.005342

In [62]:

    from sklearn.model_selection import cross_val_score, cross_val_predict
    from sklearn import metrics

    local_logit = LocalModel(weights=cd_weights,
                             pca=None,
                             local_model=LogisticRegression(C=100.0),
                             n_neighbors=200,
                             dist_tf=lambda x: 1.0 / x)

    N_CV_RUNS = 50
    scores = []
    for i in range(N_CV_RUNS):
        y_pred = cross_val_predict(local_logit, X, y, cv = StratifiedKFold(n_splits=5, shuffle=True))
        score = metrics.f1_score(y, y_pred, average='macro')
        scores.append(score)
        print "n: " + str(i) + " , f1_macro: " + str(score)

    n: 0 , f1_macro: 0.936338414356
    n: 1 , f1_macro: 0.934842733181
    n: 2 , f1_macro: 0.938304803307
    n: 3 , f1_macro: 0.935867680653
    n: 4 , f1_macro: 0.938026277023
    n: 5 , f1_macro: 0.934083539051
    n: 6 , f1_macro: 0.935414937647
    n: 7 , f1_macro: 0.938505482835
    n: 8 , f1_macro: 0.934438901779
    n: 9 , f1_macro: 0.934611086902
    n: 10 , f1_macro: 0.936246491245
    n: 11 , f1_macro: 0.9350787538
    n: 12 , f1_macro: 0.936389854692
    n: 13 , f1_macro: 0.935752888352
    n: 14 , f1_macro: 0.936895413061
    n: 15 , f1_macro: 0.936256180624
    n: 16 , f1_macro: 0.936211715387
    n: 17 , f1_macro: 0.937781672495
    n: 18 , f1_macro: 0.938247488681
    n: 19 , f1_macro: 0.935259548525
    n: 20 , f1_macro: 0.934649461347
    n: 21 , f1_macro: 0.936350359853
    n: 22 , f1_macro: 0.935780668302
    n: 23 , f1_macro: 0.937274064113
    n: 24 , f1_macro: 0.936183218652
    n: 25 , f1_macro: 0.933797687719
    n: 26 , f1_macro: 0.935882518526
    n: 27 , f1_macro: 0.937192729458
    n: 28 , f1_macro: 0.937740528375
    n: 29 , f1_macro: 0.935956143698
    n: 30 , f1_macro: 0.9385980516
    n: 31 , f1_macro: 0.93653625538
    n: 32 , f1_macro: 0.936476229607
    n: 33 , f1_macro: 0.937296033068
    n: 34 , f1_macro: 0.937330752163
    n: 35 , f1_macro: 0.936825610526
    n: 36 , f1_macro: 0.93724428216
    n: 37 , f1_macro: 0.933639619907
    n: 38 , f1_macro: 0.936110846184
    n: 39 , f1_macro: 0.936538566079
    n: 40 , f1_macro: 0.93821285848
    n: 41 , f1_macro: 0.938301999094
    n: 42 , f1_macro: 0.937843188355
    n: 43 , f1_macro: 0.934564326
    n: 44 , f1_macro: 0.933528907366
    n: 45 , f1_macro: 0.935393207831
    n: 46 , f1_macro: 0.936120553299
    n: 47 , f1_macro: 0.938824840524
    n: 48 , f1_macro: 0.936085629854
    n: 49 , f1_macro: 0.933351890871

In [66]:

    nn = experimental.KNNWeightedFeatures(weights=cd_weights)

    scores_nn = []
    for i in range(N_CV_RUNS):
        y_pred = cross_val_predict(nn, X, y, cv = StratifiedKFold(n_splits=5, shuffle=True))
        score_nn = metrics.f1_score(y, y_pred, average='macro')
        scores_nn.append(score_nn)
        print "n: " + str(i) + " , f1_macro: " + str(score_nn)

    n: 0 , f1_macro: 0.923193661343
    n: 1 , f1_macro: 0.921817989278
    n: 2 , f1_macro: 0.923130320183
    n: 3 , f1_macro: 0.924449786936
    n: 4 , f1_macro: 0.924668181686
    n: 5 , f1_macro: 0.920140886358
    n: 6 , f1_macro: 0.922941889461
    n: 7 , f1_macro: 0.923671129986
    n: 8 , f1_macro: 0.922635801472
    n: 9 , f1_macro: 0.922965996078
    n: 10 , f1_macro: 0.923246373188
    n: 11 , f1_macro: 0.923302982662
    n: 12 , f1_macro: 0.925290944876
    n: 13 , f1_macro: 0.923231219109
    n: 14 , f1_macro: 0.923113436935
    n: 15 , f1_macro: 0.923158611882
    n: 16 , f1_macro: 0.923320541748
    n: 17 , f1_macro: 0.923014928277
    n: 18 , f1_macro: 0.923776635197
    n: 19 , f1_macro: 0.92131526309
    n: 20 , f1_macro: 0.920124510345
    n: 21 , f1_macro: 0.922611643535
    n: 22 , f1_macro: 0.924169789271
    n: 23 , f1_macro: 0.924741534226
    n: 24 , f1_macro: 0.923641155914
    n: 25 , f1_macro: 0.924425932472
    n: 26 , f1_macro: 0.923481169989
    n: 27 , f1_macro: 0.925762196019
    n: 28 , f1_macro: 0.919099377721
    n: 29 , f1_macro: 0.924574916871
    n: 30 , f1_macro: 0.922985666762
    n: 31 , f1_macro: 0.921920033639
    n: 32 , f1_macro: 0.923874730089
    n: 33 , f1_macro: 0.921589089564
    n: 34 , f1_macro: 0.920896865447
    n: 35 , f1_macro: 0.921003470913
    n: 36 , f1_macro: 0.921755300173
    n: 37 , f1_macro: 0.920622296713
    n: 38 , f1_macro: 0.923407948153
    n: 39 , f1_macro: 0.922401688607
    n: 40 , f1_macro: 0.922632855385
    n: 41 , f1_macro: 0.919554235149
    n: 42 , f1_macro: 0.924547479982
    n: 43 , f1_macro: 0.922732309941
    n: 44 , f1_macro: 0.923952666262
    n: 45 , f1_macro: 0.92190582431
    n: 46 , f1_macro: 0.923188112749
    n: 47 , f1_macro: 0.923504333571
    n: 48 , f1_macro: 0.92401730911
    n: 49 , f1_macro: 0.919859082339

In [102]:

    sns.reset_orig()
    sns.boxplot( data=np.vstack((scores, scores_nn)).T, orient='vetical', color='gray')
    plt.xticks([0,1], ['local logit', 'nearest neighbor'])
    plt.grid();

In [95]:

    with open("scores_50_CV", "wb") as f:
        pickle.dump(dict(scores_50CV_local=scores, scores_50CV_nn=scores_nn), f)

In [ ]: