haplo_train.c

Go to the documentation of this file.

/*
 * This work is licensed under a Creative Commons 
 * Attribution-Noncommercial-Share Alike 3.0 United States License.
 * 
 *    http://creativecommons.org/licenses/by-nc-sa/3.0/us/
 * 
 * You are free:
 * 
 *    to Share - to copy, distribute, display, and perform the work
 *    to Remix - to make derivative works
 * 
 * Under the following conditions:
 * 
 *    Attribution. You must attribute the work in the manner specified by the
 *    author or licensor (but not in any way that suggests that they endorse you
 *    or your use of the work).
 * 
 *    Noncommercial. You may not use this work for commercial purposes.
 * 
 *    Share Alike. If you alter, transform, or build upon this work, you may
 *    distribute the resulting work only under the same or similar license to
 *    this one.
 * 
 * For any reuse or distribution, you must make clear to others the license
 * terms of this work. The best way to do this is by including this header.
 * 
 * Any of the above conditions can be waived if you get permission from the
 * copyright holder.
 * 
 * Apart from the remix rights granted under this license, nothing in this
 * license impairs or restricts the author's moral rights.
 */


#include <config.h>

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <string.h>
#include <errno.h>

#ifdef HAPLO_HAVE_DMALLOC
#include <dmalloc.h>
#endif

#include <jwsc/base/error.h>
#include <jwsc/base/option.h>
#include <jwsc/vector/vector.h>
#include <jwsc/matrix/matrix.h>
#include <jwsc/matblock/matblock.h>

#include "haplo_groups.h"
#include "options.h"
#include "input.h"
#include "nb_freq.h"
#include "nb_gauss.h"
#include "nb_gmm.h"
#include "mv_gmm.h"
#include "mv_mmm.h"
#ifdef HAPLO_ENABLE_SVM
#include "svm_tree.h"
#endif
#ifdef HAPLO_ENABLE_WEKA
#include "weka.h"
#endif
#include "nearest.h"


#ifdef HAPLO_ENABLE_SVM
#define NUM_SVM_OPTS 2
#else
#define NUM_SVM_OPTS 0
#endif

#ifdef HAPLO_ENABLE_WEKA
#define NUM_WEKA_OPTS 4
#else
#define NUM_WEKA_OPTS 0
#endif

#define  NUM_OPTS_NO_ARG    0 + NUM_SHARED_OPTS_NO_ARG
#define  NUM_OPTS_WITH_ARG  14 + NUM_SVM_OPTS + NUM_WEKA_OPTS + NUM_SHARED_OPTS_WITH_ARG


#define  LABEL_COL         1
#define  DATA_OUT_DIRNAME  0


Option_no_arg opts_no_arg[NUM_OPTS_NO_ARG];

Option_with_arg opts_with_arg[NUM_OPTS_WITH_ARG];

const char* data_out_dirname;


uint32_t get_num_opts_no_arg()
{
    return NUM_OPTS_NO_ARG;
}

uint32_t get_num_opts_with_arg()
{
    return NUM_OPTS_WITH_ARG;
}

void print_usage()
{
    fprintf(stderr, "usage: haplo-train OPTIONS [data-fname | <stdin>]\n");
    print_options(stderr, 27, NUM_OPTS_NO_ARG, opts_no_arg, NUM_OPTS_WITH_ARG,
            opts_with_arg);
}

Error* process_data_out_dir_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'data-out-dir' requires an argument");
    }
    data_out_dirname = arg;

    return NULL;
}

static void init_train_options(void)
{
    uint32_t i;

    char s_name;
    const char* l_name;
    const char* desc;

    Error* (*farg)(const char*);

    init_options(opts_no_arg, opts_with_arg);

    opts.label_col = LABEL_COL;

    i = NUM_SHARED_OPTS_WITH_ARG;
    l_name = "model-dir";
    s_name = 0;
    desc   = "Directory to put trained models in.";
    farg   = process_model_dir_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "data-out-dir";
    s_name = 0;
    desc   = "Directory to put the generated training data in for each model. The name of the model is used, so if this directory is set as the same as the model-dir, the models could be overwritten. The default is not to output the data.";
    farg   = process_data_out_dir_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq";
    s_name = 0;
    desc   = "Naive Bayes non-parametric frequency model tree information.";
    farg   = process_nb_freq_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-dtd";
    s_name = 0;
    desc   = "Validate the naive Bayes non-parametric frequency model tree information XML file with this DTD.";
    farg   = process_nb_freq_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss";
    s_name = 0;
    desc   = "Naive Bayes Gaussian model tree information.";
    farg   = process_nb_gauss_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-dtd";
    s_name = 0;
    desc   = "Validate the naive Bayes Gaussian model tree information XML file with this DTD.";
    farg   = process_nb_gauss_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm";
    s_name = 0;
    desc   = "Naive Bayes Gaussian mixture model tree information.";
    farg   = process_nb_gmm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-dtd";
    s_name = 0;
    desc   = "Validate the naive Bayes Gaussian mixture model tree information XML file with this DTD.";
    farg   = process_nb_gmm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm";
    s_name = 0;
    desc   = "Multivariate Gaussian mixture model tree information.";
    farg   = process_mv_gmm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-dtd";
    s_name = 0;
    desc   = "Validate the multivariate Gaussian mixture model tree information XML file with this DTD.";
    farg   = process_mv_gmm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-mmm";
    s_name = 0;
    desc   = "Multivariate multinomial mixture model tree information.";
    farg   = process_mv_mmm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-mmm-dtd";
    s_name = 0;
    desc   = "Validate the multivariate multinomial mixture model tree information XML file with this DTD.";
    farg   = process_mv_mmm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

#ifdef HAPLO_ENABLE_SVM
    l_name = "svm";
    s_name = 0;
    desc   = "SVM model tree information.";
    farg   = process_svm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-dtd";
    s_name = 0;
    desc   = "Validate the SVM model tree information XML file with this DTD.";
    farg   = process_svm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
#endif

#ifdef HAPLO_ENABLE_WEKA
    l_name = "weka-j48";
    s_name = 0;
    desc   = "Weka J48 model tree information.";
    farg   = process_weka_j48_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part";
    s_name = 0;
    desc   = "Weka PART model tree information.";
    farg   = process_weka_part_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-jar";
    s_name = 0;
    desc   = "Weka java archive file. Required for using the Weka algorithms.";
    farg   = process_weka_jar_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-dtd";
    s_name = 0;
    desc   = "Validate the Weka model tree information XML files with this DTD.";
    farg   = process_weka_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
#endif

    l_name = "nearest";
    s_name = 0;
    desc   = "Nearest neighbor model information.";
    farg   = process_nearest_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-dtd";
    s_name = 0;
    desc   = "Validate the nearest neighbor model information XML file with this DTD.";
    farg   = process_nearest_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
    assert(i == NUM_OPTS_WITH_ARG);
}


int main(int argc, const char** argv)
{
    int         argi;
    const char* data_fname = "/dev/stdin";
    Error*      err;

    Matblock_u8* ids    = NULL;
    Vector_u32* labels  = NULL;
    Matrix_i32* markers = NULL;

    NB_freq_model_tree*  nb_freq_tree  = NULL;
    NB_gauss_model_tree* nb_gauss_tree = NULL;
    NB_gmm_model_tree*   nb_gmm_tree   = NULL;
    MV_gmm_model_tree*   mv_gmm_tree   = NULL;
    MV_mmm_model_tree*   mv_mmm_tree   = NULL;
#ifdef HAPLO_ENABLE_SVM
    SVM_model_tree*      svm_tree      = NULL;
#endif
#ifdef HAPLO_ENABLE_WEKA
    Weka_model_tree*     j48_tree      = NULL;
    Weka_model_tree*     part_tree     = NULL;
#endif
    Nearest_model*       nearest_model = NULL;

    init_train_options();

    if ((err = process_options(argc, argv, &argi, NUM_OPTS_NO_ARG, opts_no_arg,
                    NUM_OPTS_WITH_ARG, opts_with_arg)) != NULL)
    {
        print_error_msg_exit("haplo-train", err->msg);
    }

    if ((argc - argi) == 1)
    {
        data_fname = argv[ argi ];
    }

    if ((err = read_haplo_groups(opts.labels_fname)))
    {
        print_error_msg_exit("haplo-train", err->msg);
    }

    if ((err = read_input(&ids, &labels, &markers, data_fname)))
    {
        print_error_msg_exit("haplo-train", err->msg);
    }

    if (!labels)
    {
        print_error_msg("haplo-train", NULL);
        print_error_msg_exit(data_fname, "No labels to train with");
    }

    if (opts.nb_freq_fname)
    {
        if ((err = train_nb_freq_model_tree(&nb_freq_tree, labels, markers,
                        opts.nb_freq_fname, opts.nb_freq_dtd_fname)) ||
            (err = write_nb_freq_model_tree(nb_freq_tree, opts.model_dirname)))
        {
            print_error_msg_exit("haplo-train", err->msg);
        }
        free_nb_freq_model_tree(nb_freq_tree);
    }

    if (opts.nb_gauss_fname)
    {
        if ((err = train_nb_gauss_model_tree(&nb_gauss_tree, labels, markers, 
                        opts.nb_gauss_fname, opts.nb_gauss_dtd_fname)) ||
            (err = write_nb_gauss_model_tree(nb_gauss_tree, 
                        opts.model_dirname)))
        {
            print_error_msg_exit("heplo-train", err->msg);
        }
        free_nb_gauss_model_tree(nb_gauss_tree);
    }

    if (opts.nb_gmm_fname)
    {
        if ((err = train_nb_gmm_model_tree(&nb_gmm_tree, labels, markers, 
                        opts.nb_gmm_fname, opts.nb_gmm_dtd_fname)) ||
            (err = write_nb_gmm_model_tree(nb_gmm_tree, opts.model_dirname)))
        {
            print_error_msg_exit("haplo-train", err->msg);
        }
        free_nb_gmm_model_tree(nb_gmm_tree);
    }

    if (opts.mv_gmm_fname)
    {
        if ((err = train_mv_gmm_model_tree(&mv_gmm_tree, labels, markers, 
                        opts.mv_gmm_fname, opts.mv_gmm_dtd_fname)) ||
            (err = write_mv_gmm_model_tree(mv_gmm_tree, opts.model_dirname)))
        {
            print_error_msg_exit("haplo-train", err->msg);
        }
        free_mv_gmm_model_tree(mv_gmm_tree);
    }

    if (opts.mv_mmm_fname)
    {
        if ((err = train_mv_mmm_model_tree(&mv_mmm_tree, labels, markers, 
                        opts.mv_mmm_fname, opts.mv_mmm_dtd_fname)) ||
            (err = write_mv_mmm_model_tree(mv_mmm_tree, opts.model_dirname)))
        {
            print_error_msg_exit("haplo-train", err->msg);
        }
        free_mv_mmm_model_tree(mv_mmm_tree);
    }

#ifdef HAPLO_ENABLE_SVM
    if (opts.svm_fname)
    {
        if ((err = train_svm_model_tree(&svm_tree, labels, markers,
                        opts.svm_fname, opts.svm_dtd_fname)) ||
            (err = write_svm_model_tree(svm_tree, opts.model_dirname)))
        {
            print_error_msg_exit("heplo-train", err->msg);
        }
        free_svm_model_tree(svm_tree);
        if (data_out_dirname)
        {
            if ((err = write_svm_model_tree_training_data(labels, markers,
                            opts.svm_fname, opts.svm_dtd_fname,
                            data_out_dirname)))
            {
                print_error_msg_exit("haplo-train", err->msg);
            }
        }
    }
#endif

#ifdef HAPLO_ENABLE_WEKA
    if (opts.weka_j48_fname)
    {
        if ((err = train_weka_j48_model_tree(&j48_tree, labels, markers,
                        opts.weka_j48_fname, opts.weka_dtd_fname,
                        opts.model_dirname, opts.weka_jar_fname)))
        {
            print_error_msg_exit("heplo-train", err->msg);
        }
        free_weka_model_tree(j48_tree);
    }

    if (opts.weka_part_fname)
    {
        if ((err = train_weka_part_model_tree(&part_tree, labels, markers,
                        opts.weka_part_fname, opts.weka_dtd_fname,
                        opts.model_dirname, opts.weka_jar_fname)))
        {
            print_error_msg_exit("heplo-train", err->msg);
        }
        free_weka_model_tree(part_tree);
    }
#endif

    if (opts.nearest_fname)
    {
        if ((err = train_nearest_model(&nearest_model, labels, markers,
                        opts.nearest_fname, opts.nearest_dtd_fname)) ||
            (err = write_nearest_model(nearest_model, opts.model_dirname)))
        {
            print_error_msg_exit("haplo-train", err->msg);
        }
        free_nearest_model(nearest_model);
    }

    free_matblock_u8(ids);
    free_vector_u32(labels);
    free_matrix_i32(markers);

    if (get_num_unhandled_errors() > 0)
    {
        print_error_msg_exit("haplo-train", "Unhandled errors");
    }

    return EXIT_SUCCESS;
}