import os.path
from exceptions import Exception
import numpy as np
import pandas as pd
from scandir import walk
from fnmatch import fnmatch
from parse_datetime import parse_datetime
from frq_info import IncorrectFormatException
from frq_features import FRQFeatures, FeaturesMissingException

FRQ_FOLDER = "..\\..\\data\\FRQ\\"
CSV_FILENAME = "..\\..\\training_data\\table.csv"

def data_to_csv(frq_folder, csv_filename):
    """Scan .frq files in frq_filder, and save feature matrix to csv file
    returns:
     (pd.DataFrame, missing_features_files_count, incorrect_format_files_count)

    """
    # find triples - .csv .frq .jpg
    filenames = set() # - triple filenames without extension
    for root, dirs, files in walk(frq_folder):
        f = lambda x : os.path.basename(x).split('.')[0]     
        frq_names = map(f, filter(lambda x : fnmatch(x, '*.frq'), files))
        csv_names = map(f, filter(lambda x : fnmatch(x, '*.csv'), files))
        img_names = map(f, filter(lambda x : fnmatch(x, '*.jpg'), files))
        for name in files:
            base = f(name)
            if base in frq_names and \
               base in csv_names and \
               base in img_names:
                filenames.add(os.path.join(root, base))
  
    ## make table with .frq and .csv (total_weight and axles count) data       
    data = np.empty((len(filenames), len(FRQFeatures.names) + 2), dtype=np.float64)
    bad = 0
    missing = 0
    filenames = sorted(list(filenames))
    for idx, filename in enumerate(filenames):
        if idx % 1000 == 0:
            print str(idx) + "\\" + str(len(filenames)) + " files read" 
        try:
            frq = FRQFeatures(filename + '.frq')
            dfcsv = pd.read_csv(filename + '.csv', sep = ';', decimal = ',')
            data[idx] = frq.features + [dfcsv['total_weight'][0]] + [dfcsv['axles_count'][0]]
        except FeaturesMissingException:
            missing += 1
        except IncorrectFormatException:
            bad += 1
        except Exception:
            bad += 1


    print "total .frq files count: " + str(len(filenames))
    print "missing features count: " + str(missing)
    print "files in incorrect format: " + str(bad)

    df = pd.DataFrame(data = data, columns = FRQFeatures.names + ('total_weight', 'axles_count'))
    # add filenames
    df['filename'] = filenames
    cols = df.columns.tolist()
    cols = cols[-1:] + cols[:-1] # move filename to the first position
    df = df[cols] # apply new ordering

    df.to_csv(path_or_buf=csv_filename, 
              sep=';',
              index = False,
              float_format='%.3f')

    return df, missing, bad


data_to_csv(FRQ_FOLDER, CSV_FILENAME)