In [1]:

    import sys
    sys.path.append('..\\..\\pyclf\\models')
    sys.path.append('..\\..\\pyclf\\model')
    sys.path.append('..\\..\\pyclf\\validation')
    sys.path.append('..\\..\\pyclf\\pyutils')
    sys.path.append('..\\..\\bokeh')

    import os.path
    import json
    import copy
    import time
    import pickle
    import numpy as np
    import pandas as pd

    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.decomposition import PCA
    from sklearn import cross_validation
    from sklearn import metrics
    from sklearn.cluster import KMeans

    from imblearn.under_sampling import RandomUnderSampler
    from imblearn.under_sampling import NearMiss
    from imblearn.under_sampling import CondensedNearestNeighbour
    from imblearn.under_sampling import OneSidedSelection
    from imblearn.under_sampling import NeighbourhoodCleaningRule
    from imblearn.under_sampling import TomekLinks
    from imblearn.under_sampling import ClusterCentroids
    from imblearn.under_sampling import EditedNearestNeighbours
    from imblearn.under_sampling import InstanceHardnessThreshold
    from imblearn.under_sampling import RepeatedEditedNearestNeighbours
    from imblearn.under_sampling import AllKNN

    from imblearn.over_sampling import RandomOverSampler
    from imblearn.over_sampling import SMOTE

    from imblearn.combine import SMOTETomek
    from imblearn.combine import SMOTEENN

    from imblearn.ensemble import EasyEnsemble
    from imblearn.ensemble import BalanceCascade

    from imblearn.pipeline import make_pipeline

    import matplotlib.pyplot as plt
    import matplotlib as mpl
    %matplotlib inline
    %config InlineBackend.figure_format='retina'
    import seaborn as sns

    import experimental
    import validation
    import weight
    import multiclass_metrics
    import optimization

    from experimental import KNNWeightedFeatures

    import bokeh_confmat_v2
    from bokeh.io import output_notebook, show
    output_notebook()

    #load configs
    dataset = "..//..//training_data/trainset_v3_with_attributes_and_profile_clusters.csv"

    with open('..//..//pyclf//configs/feature_headers.json', 'r') as f:
        feature_headers = json.load(f)

    with open('..//..//pyclf//configs//folds.json', 'r') as f:
        folds = json.load(f)

    with open('..//..//pyclf//configs//class_mapping.json', 'r') as f:
        mapper = json.load(f)

    mapper = mapper["loop_table"]

    #prepare data
    df = pd.read_csv(dataset, sep=';')
    df.loc[:,'class_name'] = df.loc[:,'class_name'].map( lambda x: mapper[x] if x in mapper.keys() else x)

    classes = df.class_name.unique()
    X = df[feature_headers['feature_names']].as_matrix()
    y = df.class_name.as_matrix()

Loading BokehJS ...
In [2]:

    def evaluate_sampler(sampler, silent=True):

        weights=np.array([5, 1, 1, 1, 1, 2.5] + 21*[1])
        knn = KNNWeightedFeatures(weights=weights)
        pipeline = make_pipeline(MinMaxScaler(), weight.Weight(weights=weights), sampler, knn)
        val = validation.Validation(pipeline, X, y, classes, n_folds='own', folds=folds["folds"])

        Xs, ys = sampler.fit_sample(X, y)
        if not silent:
            print sampler.__class__
            print sampler.__dict__
            print "f1 macro test, mean: ", str(val.f1_macro_test.mean()), "std: ", str(val.f1_macro_test.std())
            print "reduction from ", str(len(y)), " to ", str(len(ys)), " samples"

        return val.f1_macro_test, len(ys) / float(len(y))

    # sampler = CondensedNearestNeighbour()
    # sampler = RandomUnderSampler()
    # sampler = NeighbourhoodCleaningRule(n_neighbors=20)
    # sampler = ClusterCentroids()
    # sampler = NearMiss()
    # sampler = EditedNearestNeighbours(n_neighbors=10)

In [84]:

    scores = []
    stds = []
    reduced_by = []
    for ratio in np.arange(0.1, 1, 0.05):
        res, k = evaluate_sampler(RandomUnderSampler(ratio=ratio))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)

    plt.title("Random under sampling")
    plt.plot(reduced_by, scores)
    plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

Out[84]:

    <matplotlib.text.Text at 0x139aac88>

In [93]:

    scores = []
    stds = []
    reduced_by = []

    for n_neighbors in range(3, 20):
        res, k = evaluate_sampler(NeighbourhoodCleaningRule(n_neighbors=n_neighbors))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)

    plt.title("Neighbourhood Cleaning Rule")
    plt.plot(reduced_by, scores)
    plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

Out[93]:

    <matplotlib.text.Text at 0x1364bcf8>

In [25]:

    import under_sample_mod
    reload(under_sample_mod)

    scores = []
    stds = []
    reduced_by = []

    for n_clusters in [0.01, 0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.175, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.60]:
        res, k = evaluate_sampler(under_sample_mod.ClusterCentroidsMod(n_clusters=n_clusters,
                                                                      estimator=KMeans(n_init=10)))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)

    plt.title("Cluster centroids, float num_clusters")
    plt.plot(reduced_by, scores)
    plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

Out[25]:

    <matplotlib.text.Text at 0x126295f8>

In [27]:

    scores = []
    stds = []
    reduced_by = []

    for n_clusters in [2, 3, 5, 10, 15, 20, 25, 30, 35, 40, 50, 60, 70, 80]:
        res, k = evaluate_sampler(under_sample_mod.ClusterCentroidsMod(n_clusters=n_clusters,
                                                                      estimator=KMeans(n_init=10)))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)

    plt.title("Cluster centroids, fixed num_clusters")
    plt.plot(reduced_by, scores)
    plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

Out[27]:

    <matplotlib.text.Text at 0x1422ccf8>

In [47]:

    reload(under_sample_mod)

    scores = []
    stds = []
    reduced_by = []

    for num_samples in [10, 100, 250]:
        res, k = evaluate_sampler(under_sample_mod.RandomUnderSamplerMod(num_samples=num_samples, replacement=False))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)

    plt.title("RandomUnderSamplerMod")
    plt.plot(reduced_by, scores, 'r')
    #     plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

    scores = []
    stds = []
    reduced_by = []

    for num_samples in [0.01, 0.1, 0.25]:
        res, k = evaluate_sampler(under_sample_mod.RandomUnderSamplerMod(num_samples=num_samples, replacement=False))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)

    plt.title("RandomUnderSamplerMod")
    plt.plot(reduced_by, scores, 'b')
    #     plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

Out[47]:

    <matplotlib.text.Text at 0x11991128>

In [5]:

    import under_sample_mod
    reload(under_sample_mod)

    t1 = time.time()
    scores = []
    stds = []
    reduced_by = []

    for n_clusters in [0.01, 0.025, 0.05, 0.075, 0.10, 0.125, 0.15, 0.175, 0.20, 0.25, 0.30, 0.35, 0.40, 0.45, 0.50, 0.60, 0.70]:
        res, k = evaluate_sampler(under_sample_mod.ClusterCentroidsMod(n_clusters=n_clusters,
                                                                      estimator=KMeans(n_init=10)))
        scores.append(res.mean())
        stds.append(res.std())
        reduced_by.append(k)

    scores = np.array(scores)
    stds = np.array(stds)
    reduced_by = np.array(reduced_by)
    t2 = time.time()
    print t2  - t1

    1605.7579999

In [6]:

    plt.title("Cluster centroids, float num_clusters")
    plt.plot(reduced_by, scores)
    plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    plt.xlabel("Reduction ratio")
    plt.ylabel("F1 Macro")

Out[6]:

    <matplotlib.text.Text at 0xe357a58>

In [20]:

    import under_sample_mod
    reload(under_sample_mod)
    scores2 = []
    stds2 = []
    reduced_by2 = []

    for n_clusters in [2, 3, 5, 10, 15, 20, 25, 30, 35, 40, 50, 60, 70, 80, 100, 125, 150, 200, 250, 300, 400, 500, 1000]:
        res, k = evaluate_sampler(under_sample_mod.ClusterCentroidsMod(n_clusters=n_clusters,
                                                                      estimator=KMeans(n_init=10)))
        scores2.append(res.mean())
        stds2.append(res.std())
        reduced_by2.append(k)

    scores2 = np.array(scores2)
    stds2 = np.array(stds2)
    reduced_by2 = np.array(reduced_by2)

In [87]:

    plt.rcParams["font.family"] = "Roman"
    # plt.rcParams["font.size"] = 13

    plt.title("Nearest prototype classifier, K-Means")
    plt.plot(reduced_by2, scores2, marker='v', lw=0.6, label="Nearest protototype, absolute number of prototypes per class")
    # plt.fill_between(reduced_by2, scores2 - 2*stds2, scores2 + 2*stds2, alpha=0.2)
    plt.xlabel("Number of prototypes (K-Means centroids) / Dataset size")
    plt.ylabel("$ \mathrm{F_{macro}} $", fontsize=15)

    plt.plot(reduced_by, scores, marker='D', lw=0.6, label="Nearest protototype, relative number of prototypes per class")
    plt.plot(np.linspace(-0.02, 0.75, 10), [m.macro_fscore] * 10, '--', label="Nearest Neighbor")
    # plt.fill_between(reduced_by, scores - 2*stds, scores + 2*stds, alpha=0.2)
    # plt.xlabel("Reduction ratio")
    # plt.ylabel("F1 Macro")
    plt.xlim([-0.02, 0.75])
    plt.legend(loc="lower right")
    plt.savefig("nearest_prototype.pdf")

In [89]:

    pickle.dump(scores2, open("scores_absolute_n_clusters", "wb"))
    pickle.dump(stds2, open("stds_absolute_n_clusters", "wb"))
    pickle.dump(reduced_by2, open("reduced_by_absolute_n_clusters", "wb"))

    pickle.dump(scores, open("scores_absolute_n_clusters", "wb"))
    pickle.dump(stds, open("stds_absolute_n_clusters", "wb"))
    pickle.dump(reduced_by, open("reduced_by_absolute_n_clusters", "wb"))

In [65]:

    val.f1_macro_test.mean()

Out[65]:

    0.92116064804444786

In [68]:

    m = multiclass_metrics.MulticlassMetrics(val.y_true, val.y_pred)

In [70]:

    m.macro_fscore

Out[70]:

    0.92118718448061387

In [78]:

    np.linspace(-0.02, 0.75, 10)

Out[78]:

    array([-0.02      ,  0.06555556,  0.15111111,  0.23666667,  0.32222222,
            0.40777778,  0.49333333,  0.57888889,  0.66444444,  0.75      ])

In [ ]: