In [1]:

    import sys
    sys.path.append('..\\..\\pyclf\\models')
    sys.path.append('..\\..\\pyclf\\model')
    sys.path.append('..\\..\\pyclf\\validation')
    sys.path.append('..\\..\\pyclf\\pyutils')
    sys.path.append('..\\..\\bokeh')

    import os.path
    import json
    import copy
    import time
    import pickle
    import numpy as np
    import pandas as pd

    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.pipeline import Pipeline
    from sklearn.preprocessing import MinMaxScaler
    from sklearn.decomposition import PCA
    from sklearn import cross_validation
    from sklearn import metrics

    import matplotlib.pyplot as plt
    import matplotlib as mpl
    %matplotlib inline
    %config InlineBackend.figure_format='retina'
    import seaborn as sns

    import experimental
    import validation
    import weight
    import multiclass_metrics
    import optimization

    import bokeh_confmat_v2
    from bokeh.io import output_notebook, show
    output_notebook()

    #load configs
    dataset = "..//..//training_data//trainset_v3_with_attributes_and_profile_clusters.csv"

    with open('..//..//pyclf//configs/feature_headers.json', 'r') as f:
        feature_headers = json.load(f)

    with open('..//..//pyclf//configs//folds.json', 'r') as f:
        folds = json.load(f)

    with open('..//..//pyclf//configs//class_mapping.json', 'r') as f:
        mapper = json.load(f)

    mapper = mapper["loop_table"]

    #prepare data
    df = pd.read_csv(dataset, sep=';')
    df.loc[:,'class_name'] = df.loc[:,'class_name'].map( lambda x: mapper[x] if x in mapper.keys() else x)

    classes = df.class_name.unique()
    X = df[feature_headers['feature_names']].as_matrix()
    y = df.class_name.as_matrix()

    C:\Users\aliak_000\Anaconda2\lib\site-packages\sklearn\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
      "This module will be removed in 0.20.", DeprecationWarning)

Loading BokehJS ...
In [2]:

    folds_ind = [ (f["train_ind"], f["test_ind"]) for f in folds["folds"]]

    w = [1.0]*X.shape[1]
    clf = experimental.KNNWeightedFeatures(w)
    score_func = lambda ytrue, ypred: multiclass_metrics.MulticlassMetrics(ytrue, ypred).macro_fscore

    clf_val = experimental.CvClfTest(X, y, score_func, folds_ind)

    def func(w):
        clf_ = copy.deepcopy(clf)
        clf_.set_feature_weights(w)
        return -clf_val.evaluate(clf_)

In [3]:

    hC1 =  optimization.HillClimb(w,
                   xdomain=[(0,12) for i in range(len(w))],
                   dx=0.2,
                   max_iter=200,
                   ftol=1e-6,
                   xtol=1e-9,
                   push_when_converged=True,
                   push_when_stucked=True,
                   push_factor=0.5,
                   n_candidates=2,
                   suppress_printing=True)

    t1 = time.time()
    hC1.run(func)
    t2 = time.time()
    print "hC1 finished"
    print "time elapsed: " + str(t2-t1)

    hC1.json_dump("hC1.json")

    hC1 finished
    time elapsed: 1500.82800007

    ..\pyclf\model\optimization.py:110: UserWarning: cant jsonify _func attribute
      warnings.warn("cant jsonify " + str(k) + " attribute")

In [4]:

    hC2 =  optimization.HillClimb(w,
                   xdomain=[(0,12) for i in range(len(w))],
                   dx=0.2,
                   max_iter=200,
                   ftol=1e-6,
                   xtol=1e-9,
                   push_when_converged=True,
                   push_when_stucked=True,
                   push_factor=0.5,
                   n_candidates=5,
                   suppress_printing=True)

    t1 = time.time()
    hC2.run(func)
    t2 = time.time()
    print "hC2 finished"
    print "time elapsed: " + str(t2-t1)

    hC2.json_dump("hC2.json")

    hC2 finished
    time elapsed: 3613.57299995

In [5]:

    hC3 =  optimization.HillClimb(w,
                   xdomain=[(0,12) for i in range(len(w))],
                   dx=0.2,
                   max_iter=200,
                   ftol=1e-6,
                   xtol=1e-9,
                   push_when_converged=True,
                   push_when_stucked=True,
                   push_factor=0.5,
                   n_candidates=10,
                   suppress_printing=True)

    t1 = time.time()
    hC3.run(func)
    t2 = time.time()
    print "hC3 finished"
    print "time elapsed: " + str(t2-t1)

    hC3.json_dump("hC3.json")

    hC3 finished
    time elapsed: 7266.82999992

In [6]:

    hC4 =  optimization.HillClimb(w,
                   xdomain=[(0,12) for i in range(len(w))],
                   dx=0.2,
                   max_iter=100,
                   ftol=1e-6,
                   xtol=1e-9,
                   push_when_converged=True,
                   push_when_stucked=True,
                   push_factor=0.5,
                   n_candidates=20,
                   suppress_printing=True)

    t1 = time.time()
    hC4.run(func)
    t2 = time.time()
    print "hC4 finished"
    print "time elapsed: " + str(t2-t1)

    hC4.json_dump("hC4.json")

    hC4 finished
    time elapsed: 7217.04099989

In [7]:

    hC5 =  optimization.HillClimb(w,
                   xdomain=[(0,12) for i in range(len(w))],
                   dx=0.2,
                   max_iter=100,
                   ftol=1e-6,
                   xtol=1e-9,
                   push_when_converged=True,
                   push_when_stucked=True,
                   push_factor=0.5,
                   n_candidates=27,
                   suppress_printing=True)

    t1 = time.time()
    hC5.run(func)
    t2 = time.time()
    print "hC5 finished"
    print "time elapsed: " + str(t2-t1)

    hC5.json_dump("hC5.json")

    hC5 finished
    time elapsed: 9784.87100005

In [8]:

    fcHc1 = optimization.FirstChoiceHillClimb(x_init=[1.0]*27,
                                         mutation_k=0.1,
                                         xdomain=[(0, 7.0) for i in range(len(w))],
                                         max_iter=1000,
                                         simulated_annealing=False,
                                         temp_schedule=None,
                                         suppress_printing=True)

    t1 = time.time()
    fcHc1.run(func)
    t2 = time.time()
    print "fcHc1 finished"
    print "time elapsed: " + str(t2-t1)
    fcHc1.json_dump("fcHc1.json")

    fcHc1 finished
    time elapsed: 1809.29799986

In [9]:

    fcHc2 = optimization.FirstChoiceHillClimb(x_init=[1.0]*27,
                                         mutation_k=0.2,
                                         xdomain=[(0, 7.0) for i in range(len(w))],
                                         max_iter=1000,
                                         simulated_annealing=False,
                                         temp_schedule=None,
                                         suppress_printing=True)

    t1 = time.time()
    fcHc2.run(func)
    t2 = time.time()
    print "fcHc1 finished"
    print "time elapsed: " + str(t2-t1)
    fcHc2.json_dump("fcHc2.json")

    fcHc1 finished
    time elapsed: 1861.12100005

In [10]:

    fcHc3 = optimization.FirstChoiceHillClimb(x_init=[1.0]*27,
                                         mutation_k=0.1,
                                         xdomain=[(0, 7.0) for i in range(len(w))],
                                         max_iter=1000,
                                         simulated_annealing=True,
                                         temp_schedule=None,
                                         suppress_printing=True)

    t1 = time.time()
    fcHc3.run(func)
    t2 = time.time()
    print "fcHc3 finished"
    print "time elapsed: " + str(t2-t1)
    fcHc3.json_dump("fcHc3.json")

    fcHc3 finished
    time elapsed: 1814.11999989

In [11]:

    fcHc4 = optimization.FirstChoiceHillClimb(x_init=[1.0]*27,
                                         mutation_k=0.2,
                                         xdomain=[(0, 7.0) for i in range(len(w))],
                                         max_iter=1000,
                                         simulated_annealing=True,
                                         temp_schedule=None,
                                         suppress_printing=True)

    t1 = time.time()
    fcHc4.run(func)
    t2 = time.time()
    print "fcHc4 finished"
    print "time elapsed: " + str(t2-t1)
    fcHc4.json_dump("fcHc4.json")

    fcHc4 finished
    time elapsed: 1838.02100015

In [77]:

    l = [1.0] + [ x /10.0 for x in range(50)]   #first element - initilization value
    f = [1.0] + [ x / 10.0 for x in range(30)]
    p = [1.0] + [ x / 10.0 for x in range(30)]

    weight_grid = { "Length":     l,
                    "Curvitude":  f,
                     "Center":    f,
                     "FillF":     f,
                     "Curv1":     f,
                     "Curv2":     f,
                     "NZPD" :     f,
                     "P00"  :     p,
                     "P01"  :     p,
                     "P02"  :     p,
                     "P03"  :     p,
                     "P04"  :     p,
                     "P05"  :     p,
                     "P06"  :     p,
                     "P07"  :     p,
                     "P08"  :     p,
                     "P09"  :     p,
                     "P10"  :     p,
                     "P11"  :     p,
                     "P12"  :     p,
                     "P13"  :     p,
                     "P14"  :     p,
                     "P15"  :     p,
                     "P16"  :     p,
                     "P17"  :     p,
                     "P18"  :     p,
                     "P19"  :     p
                 }

    weight_grid = [weight_grid[fname] for fname in feature_headers["feature_names"]]
    cD1 = optimization.CoordinateDescent(weight_grid, max_iter=2)
    t1 = time.time()
    cD1.run(func)
    t2 = time.time()
    print "cD1 finished"
    print "time elapsed: " + str(t2-t1)

    cD1 finished
    time elapsed: 3128.45799994

In [78]:

    cD1.json_dump("cD1.json")

In [4]:

    reload(optimization)
    cDD =  optimization.CoordinateDescentDichotomy(
                   xdomain=[(0,2) for i in range(len(w))],
                   max_iter=3,
                   dichotomy_tol=1e-4,
                   suppress_printing=True)

    t1 = time.time()
    cDD.run(func)
    t2 = time.time()
    print "cDD finished"
    print "time elapsed: " + str(t2-t1)
    cDD.json_dump("cDD.json")

    Stopped by user
    cDD finished
    time elapsed: 5270.53699994

    ..\..\pyclf\model\optimization.py:110: UserWarning: cant jsonify _func attribute
      warnings.warn("cant jsonify " + str(k) + " attribute")

In [27]:

    optimizers = ["hC1", "hC2", "hC3", "hC4", "hC5", "fcHc1", "fcHc2", "fcHc3", "fcHc4", "cD1", "CDD"]

    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        print optimizer, optimizerLog["_fopt"]

    hC1 -0.927395204151
    hC2 -0.927879847218
    hC3 -0.92783179952
    hC4 -0.928180119306
    hC5 -0.927021377389
    fcHc1 -0.926686259985
    fcHc2 -0.92484788198
    fcHc3 -0.926086739417
    fcHc4 -0.924185370821
    cD1 -0.926692769879
    CDD -0.924632526158

In [144]:

    scores_50_runs = dict()
    scores_50_runs["optimizers"] =  ["uniform"] + optimizers
    scores_50_runs["scores"] = np.concatenate((scores_uniform.reshape(-1,1), scores.T), axis=1)
    pickle.dump(scores_50_runs, open("scores_50_runs", "wb"))

In [5]:

    optimizers = ["hC1", "hC2", "hC3", "hC4", "hC5", "fcHc1", "fcHc2", "fcHc3", "fcHc4", "cD1", "CDD"]

    for idx, optimizer in enumerate(optimizers):
        optimizerLog = json.load(open(optimizer + ".json"))
        print optimizer, optimizerLog["_fopt"], optimizerLog["_total_evals"]

    hC1 -0.927395204151 801
    hC2 -0.927879847218 2001
    hC3 -0.92783179952 4001
    hC4 -0.928180119306 4001
    hC5 -0.927021377389 5401
    fcHc1 -0.926686259985 1001
    fcHc2 -0.92484788198 1001
    fcHc3 -0.926086739417 1001
    fcHc4 -0.924185370821 1001
    cD1 -0.926692769879 1715
    CDD -0.924632526158 2472

In [9]:

    optimizerLog.keys()

Out[9]:

    [u'_func',
     u'_xopt',
     u'max_iter',
     u'xdomain',
     u'dichotomy_tol',
     u'xs_',
     u'object',
     u'_fopt',
     u'suppress_printing',
     u'_n_evals',
     u'_total_evals',
     u'order',
     u'fs_']