import os.path
from exceptions import Exception
import numpy as np
import pandas as pd
from scandir import walk
from fnmatch import fnmatch
from parse_datetime import parse_datetime
from frq_info import IncorrectFormatException
from frq_features import FRQFeatures, FeaturesMissingException


FRQ_FOLDER = "..\\data\\FRQ\\"
CSV_FILENAME = "..\\python-pickles\\features.csv"

def frq_to_csv(frq_folder, csv_filename):
    """Scan .frq files in frq_filder, and save feature matrix to csv file
    returns:
     (pd.DataFrame, missing_features_files_count, incorrect_format_files_count)

    """
    frq_filenames = []
    for root, dirs, files in walk(frq_folder):
        for name in files:
            if fnmatch(name, '*.frq'):
                frq_filenames.append(os.path.join(root,name))
            
    data = np.empty((len(frq_filenames), len(FRQFeatures.names)), dtype=np.float64)
    bad = 0
    missing = 0
    for idx, frq_filename in enumerate(frq_filenames):
        if idx % 10000 == 0:
            print str(idx) + "\\" + str(len(frq_filenames)) + " files read" 
        try:
            frq = FRQFeatures(frq_filename)
            data[idx] = frq.features
        except FeaturesMissingException:
            missing += 1
        except IncorrectFormatException:
            bad += 1


    print "total .frq files count: " + str(len(frq_filenames))
    print "missing features count: " + str(missing)
    print "files in incorrect format: " + str(bad)

    df = pd.DataFrame(data = data, columns = FRQFeatures.names)
    # add filenames
    df['filename'] = frq_filenames
    cols = df.columns.tolist()
    cols = cols[-1:] + cols[:-1] # move filename to the first position
    df = df[cols] # apply new ordering

    df.to_csv(path_or_buf=csv_filename, 
              sep=';',
              index = False,
              float_format='%.3f')

    return df, missing, bad

if __name__ == "__main__":
    print "will scan " + FRQ_FOLDER + " folder and pickle features to " + \
                                      str(CSV_FILENAME)
    df, missing, bad = frq_to_csv(FRQ_FOLDER, CSV_FILENAME)
    print str(len(df)) + " rows at csv file saved"
