haplo_test.c

Go to the documentation of this file.

/*
 * This work is licensed under a Creative Commons 
 * Attribution-Noncommercial-Share Alike 3.0 United States License.
 * 
 *    http://creativecommons.org/licenses/by-nc-sa/3.0/us/
 * 
 * You are free:
 * 
 *    to Share - to copy, distribute, display, and perform the work
 *    to Remix - to make derivative works
 * 
 * Under the following conditions:
 * 
 *    Attribution. You must attribute the work in the manner specified by the
 *    author or licensor (but not in any way that suggests that they endorse you
 *    or your use of the work).
 * 
 *    Noncommercial. You may not use this work for commercial purposes.
 * 
 *    Share Alike. If you alter, transform, or build upon this work, you may
 *    distribute the resulting work only under the same or similar license to
 *    this one.
 * 
 * For any reuse or distribution, you must make clear to others the license
 * terms of this work. The best way to do this is by including this header.
 * 
 * Any of the above conditions can be waived if you get permission from the
 * copyright holder.
 * 
 * Apart from the remix rights granted under this license, nothing in this
 * license impairs or restricts the author's moral rights.
 */


#include <config.h>

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <assert.h>
#include <inttypes.h>
#include <math.h>
#include <unistd.h>

#include <libxml/tree.h>

#ifdef HAPLO_HAVE_DMALLOC
#include <dmalloc.h>
#endif

#include <jwsc/base/error.h>
#include <jwsc/base/option.h>
#include <jwsc/base/file_io.h>
#include <jwsc/vector/vector.h>
#include <jwsc/vector/vector_math.h>
#include <jwsc/matrix/matrix.h>
#include <jwsc/matrix/matrix_math.h>
#include <jwsc/matblock/matblock.h>
#include <jwsc/stat/stat.h>

#include "haplo_groups.h"
#include "options.h"
#include "output.h"
#include "input.h"
#include "xml.h"
#include "nb_freq.h"
#include "nb_gauss.h"
#include "nb_gmm.h"
#include "mv_gmm.h"
#ifdef HAPLO_ENABLE_SVM
#include "svm_tree.h"
#endif
#ifdef HAPLO_ENABLE_WEKA
#include "weka.h"
#endif
#include "nearest.h"


#ifdef HAPLO_ENABLE_SVM
#define NUM_SVM_OPTS 8
#else
#define NUM_SVM_OPTS 0
#endif

#ifdef HAPLO_ENABLE_WEKA
#define NUM_WEKA_OPTS 16
#else
#define NUM_WEKA_OPTS 0
#endif

#define  NUM_OPTS_NO_ARG    2 + NUM_SHARED_OPTS_NO_ARG
#define  NUM_OPTS_WITH_ARG  54 + NUM_SVM_OPTS + NUM_WEKA_OPTS + NUM_SHARED_OPTS_WITH_ARG


#define  LABEL_COL                     1
#define  TANDEM                        0
#define  TEST_TYPE                     HAPLO_TEST_LEAVE_ONE_OUT
#define  NUM_CV_FOLDS                  5
#define  NUM_CV_ITERS                  10
#define  TMP_DIRNAME                   "/tmp"
#define  NB_FREQ_SUMMARY_FNAME         "/dev/stdout"
#define  NB_FREQ_DETAILS_CNT_FNAME     "/dev/null"
#define  NB_FREQ_DETAILS_PCT_FNAME     "/dev/null"
#define  NB_FREQ_CONFUSION_CNT_FNAME   "/dev/null"
#define  NB_FREQ_CONFUSION_PCT_FNAME   "/dev/null"
#define  NB_FREQ_PREDS_FNAME           "/dev/null"
#define  NB_GAUSS_SUMMARY_FNAME        "/dev/stdout"
#define  NB_GAUSS_DETAILS_CNT_FNAME    "/dev/null"
#define  NB_GAUSS_DETAILS_PCT_FNAME    "/dev/null"
#define  NB_GAUSS_CONFUSION_CNT_FNAME  "/dev/null"
#define  NB_GAUSS_CONFUSION_PCT_FNAME  "/dev/null"
#define  NB_GAUSS_PREDS_FNAME          "/dev/null"
#define  NB_GMM_SUMMARY_FNAME          "/dev/stdout"
#define  NB_GMM_DETAILS_CNT_FNAME      "/dev/null"
#define  NB_GMM_DETAILS_PCT_FNAME      "/dev/null"
#define  NB_GMM_CONFUSION_CNT_FNAME    "/dev/null"
#define  NB_GMM_CONFUSION_PCT_FNAME    "/dev/null"
#define  NB_GMM_PREDS_FNAME            "/dev/null"
#define  MV_GMM_SUMMARY_FNAME          "/dev/stdout"
#define  MV_GMM_DETAILS_CNT_FNAME      "/dev/null"
#define  MV_GMM_DETAILS_PCT_FNAME      "/dev/null"
#define  MV_GMM_CONFUSION_CNT_FNAME    "/dev/null"
#define  MV_GMM_CONFUSION_PCT_FNAME    "/dev/null"
#define  MV_GMM_PREDS_FNAME            "/dev/null"
#define  SVM_SUMMARY_FNAME             "/dev/stdout"
#define  SVM_DETAILS_CNT_FNAME         "/dev/null"
#define  SVM_DETAILS_PCT_FNAME         "/dev/null"
#define  SVM_CONFUSION_CNT_FNAME       "/dev/null"
#define  SVM_CONFUSION_PCT_FNAME       "/dev/null"
#define  SVM_PREDS_FNAME               "/dev/null"
#define  WEKA_J48_SUMMARY_FNAME        "/dev/stdout"
#define  WEKA_J48_DETAILS_CNT_FNAME    "/dev/null"
#define  WEKA_J48_DETAILS_PCT_FNAME    "/dev/null"
#define  WEKA_J48_CONFUSION_CNT_FNAME  "/dev/null"
#define  WEKA_J48_CONFUSION_PCT_FNAME  "/dev/null"
#define  WEKA_J48_PREDS_FNAME          "/dev/null"
#define  WEKA_PART_SUMMARY_FNAME       "/dev/stdout"
#define  WEKA_PART_DETAILS_CNT_FNAME   "/dev/null"
#define  WEKA_PART_DETAILS_PCT_FNAME   "/dev/null"
#define  WEKA_PART_CONFUSION_CNT_FNAME "/dev/null"
#define  WEKA_PART_CONFUSION_PCT_FNAME "/dev/null"
#define  WEKA_PART_PREDS_FNAME         "/dev/null"
#define  NEAREST_SUMMARY_FNAME         "/dev/stdout"
#define  NEAREST_DETAILS_CNT_FNAME     "/dev/null"
#define  NEAREST_DETAILS_PCT_FNAME     "/dev/null"
#define  NEAREST_CONFUSION_CNT_FNAME   "/dev/null"
#define  NEAREST_CONFUSION_PCT_FNAME   "/dev/null"
#define  NEAREST_PREDS_FNAME           "/dev/null"
#define  TANDEM_AGREE_SUMMARY_FNAME    "/dev/stdout"
#define  TANDEM_AGREE_DETAILS_CNT_FNAME  "/dev/null"
#define  TANDEM_AGREE_DETAILS_PCT_FNAME  "/dev/null"
#define  TANDEM_SUMMARY_FNAME          "/dev/stdout"
#define  TANDEM_DETAILS_CNT_FNAME      "/dev/null"
#define  TANDEM_DETAILS_PCT_FNAME      "/dev/null"
#define  TANDEM_CONFUSION_CNT_FNAME    "/dev/null"
#define  TANDEM_CONFUSION_PCT_FNAME    "/dev/null"
#define  TANDEM_PREDS_FNAME            "/dev/null"


typedef enum
{
    HAPLO_TEST_LEAVE_ONE_OUT,
    HAPLO_TEST_CROSS_VALIDATE
}
Haplo_test_type;


Option_no_arg opts_no_arg[NUM_OPTS_NO_ARG];

Option_with_arg opts_with_arg[NUM_OPTS_WITH_ARG];

static uint8_t tandem = TANDEM;

static Haplo_test_type test_type = TEST_TYPE;

static uint32_t num_cv_folds = NUM_CV_FOLDS;

static uint32_t num_cv_iters = NUM_CV_ITERS;

static const char* tmp_dirname = TMP_DIRNAME;

static const char* nb_freq_summary_fname = NB_FREQ_SUMMARY_FNAME;

static const char* nb_freq_details_cnt_fname = NB_FREQ_DETAILS_CNT_FNAME;

static const char* nb_freq_details_pct_fname = NB_FREQ_DETAILS_PCT_FNAME;

static const char* nb_freq_confusion_cnt_fname = NB_FREQ_CONFUSION_CNT_FNAME;

static const char* nb_freq_confusion_pct_fname = NB_FREQ_CONFUSION_PCT_FNAME;

static const char* nb_freq_preds_fname = NB_FREQ_PREDS_FNAME;

static const char* nb_gauss_summary_fname = NB_GAUSS_SUMMARY_FNAME;

static const char* nb_gauss_details_cnt_fname = NB_GAUSS_DETAILS_CNT_FNAME;

static const char* nb_gauss_details_pct_fname = NB_GAUSS_DETAILS_PCT_FNAME;

static const char* nb_gauss_confusion_cnt_fname = NB_GAUSS_CONFUSION_CNT_FNAME;

static const char* nb_gauss_confusion_pct_fname = NB_GAUSS_CONFUSION_PCT_FNAME;

static const char* nb_gauss_preds_fname = NB_GAUSS_PREDS_FNAME;

static const char* nb_gmm_summary_fname = NB_GMM_SUMMARY_FNAME;

static const char* nb_gmm_details_cnt_fname = NB_GMM_DETAILS_CNT_FNAME;

static const char* nb_gmm_details_pct_fname = NB_GMM_DETAILS_PCT_FNAME;

static const char* nb_gmm_confusion_cnt_fname = NB_GMM_CONFUSION_CNT_FNAME;

static const char* nb_gmm_confusion_pct_fname = NB_GMM_CONFUSION_PCT_FNAME;

static const char* nb_gmm_preds_fname = NB_GMM_PREDS_FNAME;

static const char* mv_gmm_summary_fname = MV_GMM_SUMMARY_FNAME;

static const char* mv_gmm_details_cnt_fname = MV_GMM_DETAILS_CNT_FNAME;

static const char* mv_gmm_details_pct_fname = MV_GMM_DETAILS_PCT_FNAME;

static const char* mv_gmm_confusion_cnt_fname = MV_GMM_CONFUSION_CNT_FNAME;

static const char* mv_gmm_confusion_pct_fname = MV_GMM_CONFUSION_PCT_FNAME;

static const char* mv_gmm_preds_fname = MV_GMM_PREDS_FNAME;

static const char* svm_summary_fname = SVM_SUMMARY_FNAME;

static const char* svm_details_cnt_fname = SVM_DETAILS_CNT_FNAME;

static const char* svm_details_pct_fname = SVM_DETAILS_PCT_FNAME;

static const char* svm_confusion_cnt_fname = SVM_CONFUSION_CNT_FNAME;

static const char* svm_confusion_pct_fname = SVM_CONFUSION_PCT_FNAME;

static const char* svm_preds_fname = SVM_PREDS_FNAME;

static const char* weka_j48_summary_fname = WEKA_J48_SUMMARY_FNAME;

static const char* weka_j48_details_cnt_fname = WEKA_J48_DETAILS_CNT_FNAME;

static const char* weka_j48_details_pct_fname = WEKA_J48_DETAILS_PCT_FNAME;

static const char* weka_j48_confusion_cnt_fname = WEKA_J48_CONFUSION_CNT_FNAME;

static const char* weka_j48_confusion_pct_fname = WEKA_J48_CONFUSION_PCT_FNAME;

static const char* weka_j48_preds_fname = WEKA_J48_PREDS_FNAME;

static const char* weka_part_summary_fname = WEKA_PART_SUMMARY_FNAME;

static const char* weka_part_details_cnt_fname = WEKA_PART_DETAILS_CNT_FNAME;

static const char* weka_part_details_pct_fname = WEKA_PART_DETAILS_PCT_FNAME;

static const char* weka_part_confusion_cnt_fname = WEKA_PART_CONFUSION_CNT_FNAME;

static const char* weka_part_confusion_pct_fname = WEKA_PART_CONFUSION_PCT_FNAME;

static const char* weka_part_preds_fname = WEKA_PART_PREDS_FNAME;

static const char* nearest_summary_fname = NEAREST_SUMMARY_FNAME;

static const char* nearest_details_cnt_fname = NEAREST_DETAILS_CNT_FNAME;

static const char* nearest_details_pct_fname = NEAREST_DETAILS_PCT_FNAME;

static const char* nearest_confusion_cnt_fname = NEAREST_CONFUSION_CNT_FNAME;

static const char* nearest_confusion_pct_fname = NEAREST_CONFUSION_PCT_FNAME;

static const char* nearest_preds_fname = NEAREST_PREDS_FNAME;

static const char* tandem_agree_summary_fname = TANDEM_AGREE_SUMMARY_FNAME;

static const char* tandem_agree_details_cnt_fname = TANDEM_AGREE_DETAILS_CNT_FNAME;

static const char* tandem_agree_details_pct_fname = TANDEM_AGREE_DETAILS_PCT_FNAME;

static const char* tandem_summary_fname = TANDEM_SUMMARY_FNAME;

static const char* tandem_details_cnt_fname = TANDEM_DETAILS_CNT_FNAME;

static const char* tandem_details_pct_fname = TANDEM_DETAILS_PCT_FNAME;

static const char* tandem_confusion_cnt_fname = TANDEM_CONFUSION_CNT_FNAME;

static const char* tandem_confusion_pct_fname = TANDEM_CONFUSION_PCT_FNAME;

static const char* tandem_preds_fname = TANDEM_PREDS_FNAME;


uint32_t get_num_opts_no_arg()
{
    return NUM_OPTS_NO_ARG;
}

uint32_t get_num_opts_with_arg()
{
    return NUM_OPTS_WITH_ARG;
}

void print_usage()
{
    fprintf(stderr, "usage: haplo-test OPTIONS [data-fname | <stdin>]\n");
    print_options(stderr, 27, NUM_OPTS_NO_ARG, opts_no_arg, NUM_OPTS_WITH_ARG,
            opts_with_arg);
}

static Error* process_tandem_opt()
{
    tandem = 1;
    return NULL;
}

static Error* process_test_type_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'test-type' requires an argument");
    }
    if (strncmp(arg, "loo", 13) == 0)
    {
        test_type = HAPLO_TEST_LEAVE_ONE_OUT;
    }
    else if (strncmp(arg, "cv", 2) == 0)
    {
        test_type = HAPLO_TEST_CROSS_VALIDATE;
    }
    else
    {
        return JWSC_EARG("Option 'test-type' must be one of {loo, cv}");
    }
    return NULL;
}

static Error* process_num_cv_folds_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'num-cv-folds' requires an argument");
    }
    if (sscanf(arg, "%u", &num_cv_folds) != 1 || num_cv_folds < 1)
    {
        return JWSC_EARG("Option 'num-cv-folds' must be > 0");
    }

    return NULL;
}

static Error* process_num_cv_iters_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'num-cv-iters' requires an argument");
    }
    if (sscanf(arg, "%u", &num_cv_iters) != 1 || num_cv_iters < 1)
    {
        return JWSC_EARG("Option 'num-cv-iters' must be > 0");
    }

    return NULL;
}

Error* process_tmp_dir_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tmp-dir' requires an argument");
    }
    tmp_dirname = arg;

    return NULL;
}

Error* process_nb_freq_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-freq-summary-out' requires an argument");
    }
    nb_freq_summary_fname = arg;
    return NULL;
}

Error* process_nb_freq_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-freq-details-pct-out' requires an argument");
    }
    nb_freq_details_pct_fname = arg;
    return NULL;
}

Error* process_nb_freq_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-freq-details-cnt-out' requires an argument");
    }
    nb_freq_details_cnt_fname = arg;
    return NULL;
}

Error* process_nb_freq_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-freq-confusion-cnt-out' requires an argument");
    }
    nb_freq_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_nb_freq_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-freq-confusion-pct-out' requires an argument");
    }
    nb_freq_confusion_pct_fname = arg;
    return NULL;
}

Error* process_nb_freq_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-freq-preds-out' requires an argument");
    }
    nb_freq_preds_fname = arg;
    return NULL;
}

Error* process_nb_gauss_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gauss-summary-out' requires an argument");
    }
    nb_gauss_summary_fname = arg;
    return NULL;
}

Error* process_nb_gauss_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gauss-details-pct-out' requires an argument");
    }
    nb_gauss_details_pct_fname = arg;
    return NULL;
}

Error* process_nb_gauss_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gauss-details-cnt-out' requires an argument");
    }
    nb_gauss_details_cnt_fname = arg;
    return NULL;
}

Error* process_nb_gauss_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gauss-confusion-cnt-out' requires an argument");
    }
    nb_gauss_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_nb_gauss_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gauss-confusion-pct-out' requires an argument");
    }
    nb_gauss_confusion_pct_fname = arg;
    return NULL;
}

Error* process_nb_gauss_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gauss-preds-out' requires an argument");
    }
    nb_gauss_preds_fname = arg;
    return NULL;
}

Error* process_nb_gmm_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gmm-summary-out' requires an argument");
    }
    nb_gmm_summary_fname = arg;
    return NULL;
}

Error* process_nb_gmm_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gmm-details-pct-out' requires an argument");
    }
    nb_gmm_details_pct_fname = arg;
    return NULL;
}

Error* process_nb_gmm_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gmm-details-cnt-out' requires an argument");
    }
    nb_gmm_details_cnt_fname = arg;
    return NULL;
}

Error* process_nb_gmm_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gmm-confusion-cnt-out' requires an argument");
    }
    nb_gmm_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_nb_gmm_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gmm-confusion-pct-out' requires an argument");
    }
    nb_gmm_confusion_pct_fname = arg;
    return NULL;
}

Error* process_nb_gmm_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nb-gmm-preds-out' requires an argument");
    }
    nb_gmm_preds_fname = arg;
    return NULL;
}

Error* process_mv_gmm_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'mv-gmm-summary-out' requires an argument");
    }
    mv_gmm_summary_fname = arg;
    return NULL;
}

Error* process_mv_gmm_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'mv-gmm-details-pct-out' requires an argument");
    }
    mv_gmm_details_pct_fname = arg;
    return NULL;
}

Error* process_mv_gmm_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'mv-gmm-details-cnt-out' requires an argument");
    }
    mv_gmm_details_cnt_fname = arg;
    return NULL;
}

Error* process_mv_gmm_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'mv-gmm-confusion-cnt-out' requires an argument");
    }
    mv_gmm_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_mv_gmm_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'mv-gmm-confusion-pct-out' requires an argument");
    }
    mv_gmm_confusion_pct_fname = arg;
    return NULL;
}

Error* process_mv_gmm_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'mv-gmm-preds-out' requires an argument");
    }
    mv_gmm_preds_fname = arg;
    return NULL;
}

Error* process_svm_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'svm-summary-out' requires an argument");
    }
    svm_summary_fname = arg;
    return NULL;
}

Error* process_svm_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'svm-details-pct-out' requires an argument");
    }
    svm_details_pct_fname = arg;
    return NULL;
}

Error* process_svm_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'svm-details-cnt-out' requires an argument");
    }
    svm_details_cnt_fname = arg;
    return NULL;
}

Error* process_svm_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'svm-confusion-cnt-out' requires an argument");
    }
    svm_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_svm_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'svm-confusion-pct-out' requires an argument");
    }
    svm_confusion_pct_fname = arg;
    return NULL;
}

Error* process_svm_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'svm-preds-out' requires an argument");
    }
    svm_preds_fname = arg;
    return NULL;
}

Error* process_weka_j48_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-j48-summary-out' requires an argument");
    }
    weka_j48_summary_fname = arg;
    return NULL;
}

Error* process_weka_j48_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-j48-details-pct-out' requires an argument");
    }
    weka_j48_details_pct_fname = arg;
    return NULL;
}

Error* process_weka_j48_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-j48-details-cnt-out' requires an argument");
    }
    weka_j48_details_cnt_fname = arg;
    return NULL;
}

Error* process_weka_j48_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-j48-confusion-cnt-out' requires an argument");
    }
    weka_j48_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_weka_j48_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-j48-confusion-pct-out' requires an argument");
    }
    weka_j48_confusion_pct_fname = arg;
    return NULL;
}

Error* process_weka_j48_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-j48-preds-out' requires an argument");
    }
    weka_j48_preds_fname = arg;
    return NULL;
}

Error* process_weka_part_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-part-summary-out' requires an argument");
    }
    weka_part_summary_fname = arg;
    return NULL;
}

Error* process_weka_part_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-part-details-pct-out' requires an argument");
    }
    weka_part_details_pct_fname = arg;
    return NULL;
}

Error* process_weka_part_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-part-details-cnt-out' requires an argument");
    }
    weka_part_details_cnt_fname = arg;
    return NULL;
}

Error* process_weka_part_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-part-confusion-cnt-out' requires an argument");
    }
    weka_part_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_weka_part_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-part-confusion-pct-out' requires an argument");
    }
    weka_part_confusion_pct_fname = arg;
    return NULL;
}

Error* process_weka_part_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'weka-part-preds-out' requires an argument");
    }
    weka_part_preds_fname = arg;
    return NULL;
}

Error* process_nearest_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nearest-summary-out' requires an argument");
    }
    nearest_summary_fname = arg;
    return NULL;
}

Error* process_nearest_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nearest-details-pct-out' requires an argument");
    }
    nearest_details_pct_fname = arg;
    return NULL;
}

Error* process_nearest_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nearest-details-cnt-out' requires an argument");
    }
    nearest_details_cnt_fname = arg;
    return NULL;
}

Error* process_nearest_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nearest-confusion-cnt-out' requires an argument");
    }
    nearest_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_nearest_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nearest-confusion-pct-out' requires an argument");
    }
    nearest_confusion_pct_fname = arg;
    return NULL;
}

Error* process_nearest_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'nearest-preds-out' requires an argument");
    }
    nearest_preds_fname = arg;
    return NULL;
}

Error* process_tandem_agree_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-agree-summary-out' requires an argument");
    }
    tandem_agree_summary_fname = arg;
    return NULL;
}

Error* process_tandem_agree_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-agree-details-pct-out' requires an argument");
    }
    tandem_agree_details_pct_fname = arg;
    return NULL;
}

Error* process_tandem_agree_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-agree-details-cnt-out' requires an argument");
    }
    tandem_agree_details_cnt_fname = arg;
    return NULL;
}

Error* process_tandem_summary_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-summary-out' requires an argument");
    }
    tandem_summary_fname = arg;
    return NULL;
}

Error* process_tandem_details_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-details-pct-out' requires an argument");
    }
    tandem_details_pct_fname = arg;
    return NULL;
}

Error* process_tandem_details_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-details-cnt-out' requires an argument");
    }
    tandem_details_cnt_fname = arg;
    return NULL;
}

Error* process_tandem_confusion_cnt_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-confusion-cnt-out' requires an argument");
    }
    tandem_confusion_cnt_fname = arg;
    return NULL;
}

Error* process_tandem_confusion_pct_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-confusion-pct-out' requires an argument");
    }
    tandem_confusion_pct_fname = arg;
    return NULL;
}

Error* process_tandem_preds_out_opt(Option_arg arg)
{
    if (arg == NULL)
    {
        return JWSC_EARG("Option 'tandem-preds-out' requires an argument");
    }
    tandem_preds_fname = arg;
    return NULL;
}

static void init_test_options(void)
{
    uint32_t i;

    char s_name;
    const char* l_name;
    const char* desc;

    Error* (*fnoarg)();
    Error* (*farg)(const char*);

    init_options(opts_no_arg, opts_with_arg);

    opts.label_col = LABEL_COL;

    i = NUM_SHARED_OPTS_NO_ARG;
    l_name = "tandem";
    s_name = 0;
    desc   = "Perform tandem classifier decision analysis.";
    fnoarg = process_tandem_opt;
    init_option_no_arg(&(opts_no_arg[i++]), l_name, s_name, desc, fnoarg);

    l_name = "exclude-one";
    s_name = 0;
    desc   = "When performing tandem classifier analysis, exclude at most one prediction from the set of classification algorithms. There must be three or more algorithms in play for this to take effect.";
    fnoarg = process_exclude_one_opt;
    init_option_no_arg(&(opts_no_arg[i++]), l_name, s_name, desc, fnoarg);
    assert(i == NUM_OPTS_NO_ARG);

    i = NUM_SHARED_OPTS_WITH_ARG;
    l_name = "test-type";
    s_name = 0;
    desc   = "Type of testing to use. Must be one of leave-one-out or cross-validate. Use one of the abbreviations {loo, cv}.";
    farg   = process_test_type_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tmp-dir";
    s_name = 0;
    desc   = "Directory for temporary files, including trained models.";
    farg   = process_tmp_dir_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "num-cv-folds";
    s_name = 0;
    desc   = "Number of data folds to use per cross-validation iteration.";
    farg   = process_num_cv_folds_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "num-cv-iters";
    s_name = 0;
    desc   = "Number of cross-validation iterations.";
    farg   = process_num_cv_iters_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq";
    s_name = 0;
    desc   = "Naive Bayes non-parametric frequency model tree information.";
    farg   = process_nb_freq_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-dtd";
    s_name = 0;
    desc   = "Validate the naive Bayes non-parametric frequency model tree information XML file with this DTD.";
    farg   = process_nb_freq_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss";
    s_name = 0;
    desc   = "Naive Bayes Gaussian model tree information.";
    farg   = process_nb_gauss_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-dtd";
    s_name = 0;
    desc   = "Validate the naive Bayes Gaussian model tree information XML file with this DTD.";
    farg   = process_nb_gauss_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm";
    s_name = 0;
    desc   = "Naive Bayes Gaussian mixture model tree information.";
    farg   = process_nb_gmm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-dtd";
    s_name = 0;
    desc   = "Validate the naive Bayes Gaussian mixture model tree information XML file with this DTD.";
    farg   = process_nb_gmm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm";
    s_name = 0;
    desc   = "Multivariate Gaussian mixture model tree information.";
    farg   = process_mv_gmm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-dtd";
    s_name = 0;
    desc   = "Validate the multivariate Gaussian mixture model tree information XML file with this DTD.";
    farg   = process_mv_gmm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

#ifdef HAPLO_ENABLE_SVM
    l_name = "svm";
    s_name = 0;
    desc   = "SVM model tree information.";
    farg   = process_svm_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-dtd";
    s_name = 0;
    desc   = "Validate the SVM model tree information XML file with this DTD.";
    farg   = process_svm_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
#endif

#ifdef HAPLO_ENABLE_WEKA
    l_name = "weka-j48";
    s_name = 0;
    desc   = "Weka J48 model tree information.";
    farg   = process_weka_j48_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part";
    s_name = 0;
    desc   = "Weka PART model tree information.";
    farg   = process_weka_part_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-jar";
    s_name = 0;
    desc   = "Weka java archive file. Required for using the Weka algorithms.";
    farg   = process_weka_jar_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-dtd";
    s_name = 0;
    desc   = "Validate the Weka model tree information XML files with this DTD.";
    farg   = process_weka_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
#endif

    l_name = "nearest";
    s_name = 0;
    desc   = "Nearest neighbor model information.";
    farg   = process_nearest_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-dtd";
    s_name = 0;
    desc   = "Validate the nearest neighbor model information XML file with this DTD.";
    farg   = process_nearest_dtd_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-max-d";
    s_name = 0;
    desc   = "Maximum distance allowed for a nearest neighbor classification.";
    farg   = process_nearest_max_d_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-summary-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes freqency model test performance summary to. The default is stdout.";
    farg   = process_nb_freq_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-details-cnt-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes freqency model test performance details (counts) to. The default is stdout.";
    farg   = process_nb_freq_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-details-pct-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes freqency model test performance details (percents) to. The default is stdout.";
    farg   = process_nb_freq_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes freqency model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_nb_freq_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes freqency model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_nb_freq_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-freq-preds-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes freqency model test predictions to. The default is no output.";
    farg   = process_nb_freq_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-summary-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian model test performance summary to. The default is stdout.";
    farg   = process_nb_gauss_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-details-cnt-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian model test performance details (counts) to. The default is stdout.";
    farg   = process_nb_gauss_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-details-pct-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian model test performance details (percents) to. The default is stdout.";
    farg   = process_nb_gauss_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_nb_gauss_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_nb_gauss_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gauss-preds-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian model test predictions to. The default is no output.";
    farg   = process_nb_gauss_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-summary-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian mixture model test performance summary to. The default is stdout.";
    farg   = process_nb_gmm_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-details-cnt-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian mixture model test performance details (counts) to. The default is stdout.";
    farg   = process_nb_gmm_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-details-pct-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian mixture model test performance details (percents) to. The default is stdout.";
    farg   = process_nb_gmm_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian mixture model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_nb_gmm_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian mixture model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_nb_gmm_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nb-gmm-preds-out";
    s_name = 0;
    desc   = "File to output the Naive Bayes Gaussian mixture model test predictions to. The default is no output.";
    farg   = process_nb_gmm_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-summary-out";
    s_name = 0;
    desc   = "File to output the multivariate Gaussian mixture model test performance summary to. The default is stdout.";
    farg   = process_mv_gmm_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-details-cnt-out";
    s_name = 0;
    desc   = "File to output the multivariate Gaussian mixture model test performance details (counts) to. The default is stdout.";
    farg   = process_mv_gmm_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-details-pct-out";
    s_name = 0;
    desc   = "File to output the multivariate Gaussian mixture model test performance details (percents) to. The default is stdout.";
    farg   = process_mv_gmm_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the multivariate Gaussian mixture model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_mv_gmm_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the multivariate Gaussian mixture model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_mv_gmm_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "mv-gmm-preds-out";
    s_name = 0;
    desc   = "File to output the multivariate Gaussian mixture model test predictions to. The default is no output.";
    farg   = process_mv_gmm_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

#ifdef HAPLO_ENABLE_SVM
    l_name = "svm-summary-out";
    s_name = 0;
    desc   = "File to output the SVM model test performance summary to. The default is stdout.";
    farg   = process_svm_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-details-cnt-out";
    s_name = 0;
    desc   = "File to output the SVM model test performance details (counts) to. The default is stdout.";
    farg   = process_svm_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-details-pct-out";
    s_name = 0;
    desc   = "File to output the SVM model test performance details (percents) to. The default is stdout.";
    farg   = process_svm_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the SVM model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_svm_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the SVM model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_svm_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "svm-preds-out";
    s_name = 0;
    desc   = "File to output the SVM model test predictions to. The default is no output.";
    farg   = process_svm_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
#endif

#ifdef HAPLO_ENABLE_WEKA
    l_name = "weka-j48-summary-out";
    s_name = 0;
    desc   = "File to output the Weka J48 model test performance summary to. The default is stdout.";
    farg   = process_weka_j48_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-j48-details-cnt-out";
    s_name = 0;
    desc   = "File to output the Weka J48 model test performance details (counts) to. The default is stdout.";
    farg   = process_weka_j48_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-j48-details-pct-out";
    s_name = 0;
    desc   = "File to output the Weka J48 model test performance details (percents) to. The default is stdout.";
    farg   = process_weka_j48_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-j48-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the Weka J48 model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_weka_j48_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-j48-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the Weka J48 model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_weka_j48_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-j48-preds-out";
    s_name = 0;
    desc   = "File to output the Weka J48 model test predictions to. The default is no output.";
    farg   = process_weka_j48_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part-summary-out";
    s_name = 0;
    desc   = "File to output the Weka PART model test performance summary to. The default is stdout.";
    farg   = process_weka_part_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part-details-cnt-out";
    s_name = 0;
    desc   = "File to output the Weka PART model test performance details (counts) to. The default is stdout.";
    farg   = process_weka_part_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part-details-pct-out";
    s_name = 0;
    desc   = "File to output the Weka PART model test performance details (percents) to. The default is stdout.";
    farg   = process_weka_part_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the Weka PART model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_weka_part_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the Weka PART model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_weka_part_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "weka-part-preds-out";
    s_name = 0;
    desc   = "File to output the Weka PART model test predictions to. The default is no output.";
    farg   = process_weka_part_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
#endif

    l_name = "nearest-summary-out";
    s_name = 0;
    desc   = "File to output the nearest neighbor model test performance summary to. The default is stdout.";
    farg   = process_nearest_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-details-cnt-out";
    s_name = 0;
    desc   = "File to output the nearest neighbor model test performance details (counts) to. The default is stdout.";
    farg   = process_nearest_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-details-pct-out";
    s_name = 0;
    desc   = "File to output the nearest neighbor model test performance details (percents) to. The default is stdout.";
    farg   = process_nearest_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the nearest neighbor model test confusion matrix (counts) to. The default is stdout.";
    farg   = process_nearest_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the nearest neighbor model test confusion matrix (percents) to. The default is stdout.";
    farg   = process_nearest_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "nearest-preds-out";
    s_name = 0;
    desc   = "File to output the nearest neighbor model test predictions to. The default is no output.";
    farg   = process_nearest_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-agree-summary-out";
    s_name = 0;
    desc   = "File to output the tandem agreement test performance summary to. The default is stdout.";
    farg   = process_tandem_agree_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-agree-details-cnt-out";
    s_name = 0;
    desc   = "File to output the tandem agreement test performance details (counts) to. The default is stdout.";
    farg   = process_tandem_agree_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-agree-details-pct-out";
    s_name = 0;
    desc   = "File to output the tandem agreement test performance details (percents) to. The default is stdout.";
    farg   = process_tandem_agree_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-summary-out";
    s_name = 0;
    desc   = "File to output the tandem test performance summary to. The default is stdout.";
    farg   = process_tandem_summary_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-details-cnt-out";
    s_name = 0;
    desc   = "File to output the tandem test performance details (counts) to. The default is stdout.";
    farg   = process_tandem_details_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-details-pct-out";
    s_name = 0;
    desc   = "File to output the tandem test performance details (percents) to. The default is stdout.";
    farg   = process_tandem_details_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-confusion-cnt-out";
    s_name = 0;
    desc   = "File to output the tandem test confusion matrix (counts) to. The default is stdout.";
    farg   = process_tandem_confusion_cnt_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-confusion-pct-out";
    s_name = 0;
    desc   = "File to output the tandem test confusion matrix (percents) to. The default is stdout.";
    farg   = process_tandem_confusion_pct_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);

    l_name = "tandem-preds-out";
    s_name = 0;
    desc   = "File to output the tandem test predictions to. The default is no output.";
    farg   = process_tandem_preds_out_opt;
    init_option_with_arg(&(opts_with_arg[i++]), l_name, s_name, desc, farg);
    assert(i == NUM_OPTS_WITH_ARG);
}

static uint8_t num_models_to_test()
{
    return (opts.nb_freq_fname   != 0) +
           (opts.nb_gauss_fname  != 0) +
           (opts.nb_gmm_fname    != 0) +
           (opts.mv_gmm_fname    != 0) +
#ifdef HAPLO_ENABLE_SVM
           (opts.svm_fname       != 0) +
#endif
#ifdef HAPLO_ENABLE_WEKA
           (opts.weka_j48_fname  != 0) +
           (opts.weka_part_fname != 0) +
#endif
           (opts.nearest_fname   != 0);
}

static void find_ancestors
(
    Vector_u32**      ancestor_types_out,
    Vector_u32**      ancestor_labels_out,
    const Vector_u32* labels_1,
    const Vector_u32* labels_2
)
{
    uint32_t i;
    uint32_t ancestor_label;
    Haplo_ancestor_type ancestor_type;

    if (!labels_1 || !labels_2)
        return;

    assert(labels_1->num_elts == labels_2->num_elts);
    create_vector_u32(ancestor_types_out, labels_1->num_elts);
    create_vector_u32(ancestor_labels_out, labels_1->num_elts);

    for (i = 0; i < labels_1->num_elts; i++)
    {
        ancestor_type = find_ancestor_index_of_pair(&ancestor_label,
                labels_1->elts[ i ], labels_2->elts[ i ]);

        (*ancestor_types_out)->elts[ i ] = ancestor_type;
        (*ancestor_labels_out)->elts[ i ] = ancestor_label;
    }
}

static void find_tandem_ancestors
(
    Vector_u32**      ancestor_types_out,
    Vector_u32**      ancestor_labels_out,
    const Vector_u32* tandem_types,
    const Vector_u32* tandem_labels,
    const Vector_u32* test_labels
)
{
    uint32_t i;
    uint32_t ancestor_label;
    Haplo_ancestor_type ancestor_type;

    assert(tandem_labels->num_elts == test_labels->num_elts);
    create_vector_u32(ancestor_types_out, tandem_labels->num_elts);
    create_vector_u32(ancestor_labels_out, tandem_labels->num_elts);

    for (i = 0; i < tandem_labels->num_elts; i++)
    {
        if (tandem_types->elts[ i ] != HAPLO_ANCESTOR_NONE)
        {
            ancestor_type = find_ancestor_index_of_pair(&ancestor_label,
                    tandem_labels->elts[ i ], test_labels->elts[ i ]);
        }
        else 
        {
            ancestor_type = HAPLO_ANCESTOR_NONE;
            ancestor_label = 0;
        }

        (*ancestor_types_out)->elts[ i ] = ancestor_type;
        (*ancestor_labels_out)->elts[ i ] = ancestor_label;
    }
}

static void find_ancestors_of_sets
(
    Vector_u32**      ancestor_types_out,
    Vector_u32**      ancestor_labels_out,
    const Vector_u32* labels_1,
    const Vector_u32* labels_2,
    const Vector_u32* labels_3,
    const Vector_u32* labels_4,
    const Vector_u32* labels_5,
    const Vector_u32* labels_6,
    const Vector_u32* labels_7,
    const Vector_u32* labels_8
)
{
    uint32_t            n, nn, N;
    uint32_t            i;
    uint32_t            num_labels;
    uint32_t            ancestor_label;
    Haplo_ancestor_type ancestor_type;
    Vector_u32*         labels;
    Vector_u32*         labelss;

    N = 0;

    if (labels_1) {N++; num_labels = labels_1->num_elts;}
    if (labels_2) {N++; num_labels = labels_2->num_elts;}
    if (labels_3) {N++; num_labels = labels_3->num_elts;}
    if (labels_4) {N++; num_labels = labels_4->num_elts;}
    if (labels_5) {N++; num_labels = labels_5->num_elts;}
    if (labels_6) {N++; num_labels = labels_6->num_elts;}
    if (labels_7) {N++; num_labels = labels_7->num_elts;}
    if (labels_8) {N++; num_labels = labels_8->num_elts;}

    assert(N > 0);

    labels = NULL;
    create_vector_u32(&labels, N);

    create_vector_u32(ancestor_types_out, num_labels);
    create_vector_u32(ancestor_labels_out, num_labels);

    for (i = 0; i < num_labels; i++)
    {
        N = 0;

        if (labels_1) labels->elts[ N++ ] = labels_1->elts[ i ];
        if (labels_2) labels->elts[ N++ ] = labels_2->elts[ i ];
        if (labels_3) labels->elts[ N++ ] = labels_3->elts[ i ];
        if (labels_4) labels->elts[ N++ ] = labels_4->elts[ i ];
        if (labels_5) labels->elts[ N++ ] = labels_5->elts[ i ];
        if (labels_6) labels->elts[ N++ ] = labels_6->elts[ i ];
        if (labels_7) labels->elts[ N++ ] = labels_7->elts[ i ];
        if (labels_8) labels->elts[ N++ ] = labels_8->elts[ i ];

        ancestor_type = find_ancestor_index_of_set(&ancestor_label, labels);

        if (opts.exclude_one && ancestor_type == HAPLO_ANCESTOR_NONE && N > 3)
        {
            labelss = NULL;
            create_vector_u32(&labelss, N-1);
            for (n = 0; ancestor_type == HAPLO_ANCESTOR_NONE && n < N; n++)
            {
                for (nn = 0; nn < N-1; nn++)
                {
                    if (nn < n)
                    {
                        labelss->elts[ nn ] = labels->elts[ nn ];
                    }
                    else
                    {
                        labelss->elts[ nn ] = labels->elts[ nn+1 ];
                    }
                }

                ancestor_type = find_ancestor_index_of_set(&ancestor_label,
                        labelss);
            }
            free_vector_u32(labelss);
        }

        (*ancestor_types_out)->elts[ i ] = ancestor_type;
        (*ancestor_labels_out)->elts[ i ] = ancestor_label;
    }
}

static void write_leave_one_out_summary
(
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Vector_u32*  ancestor_types,
    const Vector_u32*  ancestor_labels,
    const Vector_u32*  pred_labels,
    const Vector_d*    pred_confs,
    const Vector_u32*  tandem_types,
    const char*        fname
)
{
    uint32_t    i;
    uint32_t    n, N;
    float       D;
    const char* fmt;
    FILE*       fp;
    xmlDoc*     xml_doc  = NULL;
    xmlNode*    xml_root = NULL;
    xmlNode*    xml_node[3];
    char        xml_buf[256];
    Error*      err;
    Vector_f*   counts = NULL;

    if ((err = open_output(&fp, &xml_doc, "haplo-test-loo-summary-out",
                    "haplo-test-loo-summary-out.dtd", fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    create_zero_vector_f(&counts, 3);

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fmt = "%-10s  %-10s  %-10s\n";
                break;
            case HAPLO_OUTPUT_CSV:
                fmt = "%s,%s,%s\n";
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(fp, fmt, "Direct", "Indirect", "None");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    N = ancestor_types->num_elts;

    for (n = 0; n < N; n++)
    {
        if (!tandem_types || tandem_types->elts[n] != HAPLO_ANCESTOR_NONE)
        {
            switch (ancestor_types->elts[ n ])
            {
                case HAPLO_ANCESTOR_DIRECT:
                    counts->elts[0]++;
                    break;
                case HAPLO_ANCESTOR_INDIRECT:
                    counts->elts[1]++;
                    break;
                case HAPLO_ANCESTOR_NONE:
                    counts->elts[2]++;
                    break;
            }
        }
    }

    switch (opts.output_format)
    {
        case HAPLO_OUTPUT_TXT:
            fmt = "%-10.0f  %-10.0f  %-10.0f\n";
            fprintf(fp, fmt, counts->elts[0], counts->elts[1], counts->elts[2]);
            break;
        case HAPLO_OUTPUT_CSV:
            fmt = "%.0f,%.0f,%.0f\n";
            fprintf(fp, fmt, counts->elts[0], counts->elts[1], counts->elts[2]);
            break;
        case HAPLO_OUTPUT_XML:
            xml_node[0] = XMLNewChild(xml_root, "direct", NULL);
            xml_node[1] = XMLNewChild(xml_root, "indirect", NULL);
            xml_node[2] = XMLNewChild(xml_root, "none", NULL);
            for (i = 0; i < 3; i++)
            {
                snprintf(xml_buf, 256, "%.0f", counts->elts[ i ]);
                XMLNewChild(xml_node[ i ], "count", xml_buf);
            }
            break;
    }

    D = counts->elts[0] + counts->elts[1] + counts->elts[2];
    if (D > 0)
    {
        multiply_vector_by_scalar_f(&counts, counts, 1/D);
    }
    switch (opts.output_format)
    {
        case HAPLO_OUTPUT_TXT:
            fmt = "%-10.3f  %-10.3f  %-10.3f\n";
            fprintf(fp, fmt, counts->elts[0], counts->elts[1], counts->elts[2]);
            break;
        case HAPLO_OUTPUT_CSV:
            fmt = "%.3f,%.3f,%.3f\n";
            fprintf(fp, fmt, counts->elts[0], counts->elts[1], counts->elts[2]);
            break;
        case HAPLO_OUTPUT_XML:
            for (i = 0; i < 3; i++)
            {
                snprintf(xml_buf, 256, "%.3f", counts->elts[ i ]);
                XMLNewChild(xml_node[ i ], "percent", xml_buf);
            }
            break;
    }

    free_vector_f(counts);

    if ((err = close_output(fp, xml_doc, fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void write_leave_one_out_details
(
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Vector_u32*  ancestor_types,
    const Vector_u32*  ancestor_labels,
    const Vector_u32*  pred_labels,
    const Vector_d*    pred_confs,
    const Vector_u32*  tandem_types,
    const char*        cnt_fname,
    const char*        pct_fname
)
{
    uint32_t    i;
    uint32_t    n, nn, N;
    float       D;
    const char* label;
    const char* fmt;
    FILE*       cnt_fp;
    FILE*       pct_fp;
    xmlDoc*     xml_doc   = NULL;
    xmlNode*    xml_root  = NULL;
    xmlNode*    xml_node  = NULL;
    xmlNode*    xml_child[3] = {0};
    char        xml_buf[256];
    Error*      err;

    Vector_f* counts[3] = {0};

    if (!pred_labels)
        return;

    if ((err = open_output(&cnt_fp, &xml_doc, "haplo-test-loo-details-out",
                    "haplo-test-loo-details-out.dtd", cnt_fname)) ||
        (err = open_output(&pct_fp, NULL, NULL, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    N = get_num_haplo_groups();
    for (i = 0; i < 3; i++)
    {
        create_zero_vector_f(&(counts[ i ]), N);
    }

    for (i = 0; i < ancestor_types->num_elts; i++)
    {
        if (!tandem_types || tandem_types->elts[i] != HAPLO_ANCESTOR_NONE)
        {
            assert(pred_labels->elts[ i ] < N);
            n = pred_labels->elts[ i ];

            switch (ancestor_types->elts[ i ])
            {
                case HAPLO_ANCESTOR_DIRECT:
                    counts[0]->elts[ n ]++;
                    break;
                case HAPLO_ANCESTOR_INDIRECT:
                    counts[1]->elts[ n ]++;
                    break;
                case HAPLO_ANCESTOR_NONE:
                    counts[2]->elts[ n ]++;
                    break;
            }
        }
    }

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fmt = "%-10s  %-15s  %-15s  %-15s\n";
                break;
            case HAPLO_OUTPUT_CSV:
                fmt = "%s,%s,%s,%s\n";
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(cnt_fp, fmt, "Predicted", "Direct", "Indirect", "None");
        fprintf(pct_fp, fmt, "Predicted", "Direct", "Indirect", "None");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    nn = 0;
    for (n = 0; n < N; n++)
    {
        D = counts[0]->elts[n] + counts[1]->elts[n] + counts[2]->elts[n];
        if (D > 0)
        {
            lookup_haplo_group_label_from_index(&label, n);
            switch (opts.output_format)
            {
                case HAPLO_OUTPUT_TXT:
                    fprintf(cnt_fp, 
                         "%-10s  %-15.0f  %-15.0f  %-15.0f\n",
                            label, counts[0]->elts[n], counts[1]->elts[n],
                            counts[2]->elts[n]);
                    fprintf(pct_fp, 
                         "%-10s  %-15.3f  %-15.3f  %-15.3f\n",
                            label, counts[0]->elts[n]/D, counts[1]->elts[n]/D,
                            counts[2]->elts[n]/D);
                    break;
                case HAPLO_OUTPUT_CSV:
                    fprintf(cnt_fp, "%s,%.0f,%.0f,%.0f\n", label,
                            counts[0]->elts[n], counts[1]->elts[n],
                            counts[2]->elts[n]);
                    fprintf(pct_fp, "%s,%.3f,%.3f,%.3f\n", label,
                            counts[0]->elts[n]/D, counts[1]->elts[n]/D,
                            counts[2]->elts[n]/D);
                    break;
                case HAPLO_OUTPUT_XML:
                    xml_node = XMLNewChild(xml_root, "predicted", NULL);
                    snprintf(xml_buf, 256, "%d", nn+1);
                    XMLNewProp(xml_node, "number", xml_buf);
                    XMLNewChild(xml_node, "label", label);
                    xml_child[0] = XMLNewChild(xml_node, "direct", 0);
                    xml_child[1] = XMLNewChild(xml_node, "indirect", 0);
                    xml_child[2] = XMLNewChild(xml_node, "none", 0);
                    for (i = 0; i < 3; i++)
                    {
                        snprintf(xml_buf, 256, "%.0f", counts[i]->elts[n]);
                        XMLNewChild(xml_child[ i ], "count", xml_buf);
                        snprintf(xml_buf, 256, "%.3f", counts[i]->elts[n]/D);
                        XMLNewChild(xml_child[ i ], "percent", xml_buf);
                    }
                    break;
            }

            nn++;
        }
    }

    for (i = 0; i < 3; i++)
    {
        free_vector_f(counts[ i ]);
    }

    if ((err = close_output(cnt_fp, xml_doc, cnt_fname)) ||
        (err = close_output(pct_fp, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void write_leave_one_out_confusion
(
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Vector_u32*  ancestor_types,
    const Vector_u32*  ancestor_labels,
    const Vector_u32*  pred_labels,
    const Vector_d*    pred_confs,
    const Vector_u32*  tandem_types,
    const char*        cnt_fname,
    const char*        pct_fname
)
{
    uint32_t    i;
    uint32_t    n_1, n_2, N;
    uint32_t    nn_1, nn_2;
    float       D;
    const char* label;
    FILE*       cnt_fp;
    FILE*       pct_fp;
    xmlDoc*     xml_doc    = NULL;
    xmlNode*    xml_root   = NULL;
    xmlNode*    xml_actual = NULL;
    xmlNode*    xml_pred   = NULL;
    char        xml_buf[256];
    Error*      err;

    Vector_f* actual_counts    = NULL;
    Vector_f* predicted_counts = NULL;
    Matrix_f* confusion        = NULL;
    Matrix_f* confusion_pct    = NULL;

    if (!pred_labels)
        return;

    assert(data_labels->num_elts == pred_labels->num_elts);

    if ((err = open_output(&cnt_fp, &xml_doc, "haplo-test-loo-confusion-out",
                    "haplo-test-loo-confusion-out.dtd", cnt_fname)) ||
        (err = open_output(&pct_fp, NULL, NULL, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    N = get_num_haplo_groups();
    create_zero_vector_f(&actual_counts, N);
    create_zero_vector_f(&predicted_counts, N);
    create_zero_matrix_f(&confusion, N, N);
    create_zero_matrix_f(&confusion_pct, N, N);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        if (!tandem_types || tandem_types->elts[i] != HAPLO_ANCESTOR_NONE)
        {
        assert(data_labels->elts[ i ] < N);
        assert(pred_labels->elts[ i ] < N);

        actual_counts->elts[ data_labels->elts[ i ] ]++;
        predicted_counts->elts[ pred_labels->elts[ i ] ]++;
        confusion->elts[ data_labels->elts[ i ] ][ pred_labels->elts[ i ] ]++;
        }
    }

    copy_matrix_f(&confusion_pct, confusion);
    for (n_2 = 0; n_2 < N; n_2++)
    {
        D = predicted_counts->elts[ n_2 ];
        if (D > 0)
        {
            for (n_1 = 0; n_1 < N; n_1++)
            {
                confusion_pct->elts[ n_1 ][ n_2 ] /= D;
            }
        }
    }

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fprintf(cnt_fp, "%-10s", "Actual");
                fprintf(pct_fp, "%-10s", "Actual");
                for (n_2 = 0; n_2 < N; n_2++)
                {
                    if (predicted_counts->elts[ n_2 ])
                    {
                        lookup_haplo_group_label_from_index(&label, n_2);
                        fprintf(cnt_fp, "  %-10s", label);
                        fprintf(pct_fp, "  %-10s", label);
                    }
                }
                break;
            case HAPLO_OUTPUT_CSV:
                fprintf(cnt_fp, "%s", "Actual");
                fprintf(pct_fp, "%s", "Actual");
                for (n_2 = 0; n_2 < N; n_2++)
                {
                    if (predicted_counts->elts[ n_2 ])
                    {
                        lookup_haplo_group_label_from_index(&label, n_2);
                        fprintf(cnt_fp, ",%s", label);
                        fprintf(pct_fp, ",%s", label);
                    }
                }
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(cnt_fp, "\n");
        fprintf(pct_fp, "\n");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    nn_1 = 0;
    for (n_1 = 0; n_1 < N; n_1++)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                if (actual_counts->elts[ n_1 ] > 0)
                {
                    lookup_haplo_group_label_from_index(&label, n_1);
                    fprintf(cnt_fp, "%-10s", label);
                    fprintf(pct_fp, "%-10s", label);

                    nn_2 = 0;
                    for (n_2 = 0; n_2 < N; n_2++)
                    {
                        if (predicted_counts->elts[ n_2 ] > 0)
                        {
                            fprintf(cnt_fp, "  %-10.0f", 
                                    confusion->elts[ n_1 ][ n_2 ]);
                            fprintf(pct_fp, "  %-10.3f", 
                                    confusion_pct->elts[ n_1 ][ n_2 ]);
                            nn_2++;
                        }
                    }
                    fprintf(cnt_fp, "\n");
                    fprintf(pct_fp, "\n");
                    nn_1++;
                }
                break;
            case HAPLO_OUTPUT_CSV:
                if (actual_counts->elts[ n_1 ] > 0)
                {
                    lookup_haplo_group_label_from_index(&label, n_1);
                    fprintf(cnt_fp, "%s", label);
                    fprintf(pct_fp, "%s", label);

                    nn_2 = 0;
                    for (n_2 = 0; n_2 < N; n_2++)
                    {
                        if (predicted_counts->elts[ n_2 ] > 0)
                        {
                            fprintf(cnt_fp, ",%.0f", 
                                    confusion->elts[ n_1 ][ n_2 ]);
                            fprintf(pct_fp, ",%.3f", 
                                    confusion_pct->elts[ n_1 ][ n_2 ]);
                            nn_2++;
                        }
                    }
                    fprintf(cnt_fp, "\n");
                    fprintf(pct_fp, "\n");
                    nn_1++;
                }
                break;
            case HAPLO_OUTPUT_XML:
                if (actual_counts->elts[ n_1 ] > 0)
                {
                    lookup_haplo_group_label_from_index(&label, n_1);
                    xml_actual = XMLNewChild(xml_root, "actual", NULL);
                    snprintf(xml_buf, 256, "%d", nn_1+1);
                    XMLNewProp(xml_actual, "number", xml_buf);
                    XMLNewChild(xml_actual, "label", label);

                    nn_2 = 0;
                    for (n_2 = 0; n_2 < N; n_2++)
                    {
                        if (predicted_counts->elts[ n_2 ] > 0)
                        {
                            lookup_haplo_group_label_from_index(&label, n_2);
                            xml_pred = XMLNewChild(xml_actual, "predicted", 0);
                            snprintf(xml_buf, 256, "%d", nn_2+1);
                            XMLNewProp(xml_pred, "number", xml_buf);
                            XMLNewChild(xml_pred, "label", label);

                            snprintf(xml_buf, 256, "%.0f", 
                                    confusion->elts[n_1][n_2]);
                            XMLNewChild(xml_pred, "count", xml_buf);

                            snprintf(xml_buf, 256, "%.3f", 
                                    confusion_pct->elts[n_1][n_2]);
                            XMLNewChild(xml_pred, "percent", xml_buf);

                            nn_2++;
                        }
                    }
                    nn_1++;
                }
                break;
        }
    }

    free_vector_f(actual_counts);
    free_vector_f(predicted_counts);
    free_matrix_f(confusion);
    free_matrix_f(confusion_pct);

    if ((err = close_output(cnt_fp, xml_doc, cnt_fname)) ||
        (err = close_output(pct_fp, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void write_leave_one_out_preds
(
    const char*        type,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Vector_u32*  ancestor_types,
    const Vector_u32*  ancestor_labels,
    const Vector_u32*  pred_labels,
    const Vector_d*    pred_confs,
    const Vector_u32*  tandem_types,
    const char*        fname
)
{
    uint32_t    i, j;
    FILE*       fp;
    xmlDoc*     xml_doc    = NULL;
    xmlNode*    xml_root   = NULL;
    xmlNode*    xml_node   = NULL;
    char        xml_buf[256] = {0};
    Error*      err;

    if (!pred_labels)
        return;

    if ((err = open_output(&fp, &xml_doc, "haplo-test-loo-predictions-out",
                    "haplo-test-loo-predictions-out.dtd", fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        if (data_ids)
        {
            for (j = 0; j < data_ids->num_rows; j++)
            {
                switch (opts.output_format)
                {
                    case HAPLO_OUTPUT_TXT:
                        fprintf(fp, "ID %-7d  ", j+1);
                        break;
                    case HAPLO_OUTPUT_CSV:
                        fprintf(fp, "ID %d,", j+1);
                        break;
                    case HAPLO_OUTPUT_XML:
                        break;
                }
            }
        }

        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fprintf(fp, "%-10s  %-4s  %-10s", "Actual", "Ancestor", "Type");
                if (pred_labels) 
                    fprintf(fp, "  %-10s  %-5s", "Prediction", "Conf");
                break;
            case HAPLO_OUTPUT_CSV:
                fprintf(fp, "%s,%s,%s", "Actual", "Ancestor", "Type");
                if (pred_labels) 
                    fprintf(fp, ",%s,%s", "Prediction", "Conf");
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(fp, "\n");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    for (i = 0; i < data_labels->num_elts; i++)
    {
        if (!tandem_types || tandem_types->elts[i] != HAPLO_ANCESTOR_NONE)
        {
            if (opts.output_format == HAPLO_OUTPUT_XML)
            {
                xml_node = XMLNewChild(xml_root, "sample", NULL);
                snprintf(xml_buf, 256, "%d", i+1);
                XMLNewProp(xml_node, "number", xml_buf);
            }

            write_ids(data_ids, i, HAPLO_SEP_SUFFIX, fp, xml_node);
            write_label(data_labels, i, HAPLO_SEP_SUFFIX, fp, xml_node);

            write_ancestor_label(ancestor_types, ancestor_labels, i, 
                    HAPLO_SEP_NONE, fp, xml_node);

            write_prediction(type, pred_labels, pred_confs, i, 
                    HAPLO_SEP_PREFIX, fp, xml_node);

            if (opts.output_format != HAPLO_OUTPUT_XML)
            {
                fprintf(fp, "\n");
            }
        }
    }

    if ((err = close_output(fp, xml_doc, fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void create_leave_one_out_train_and_test_data
(
    Vector_u32**       train_labels_out,
    Matrix_i32**       train_markers_out,
    Matrix_i32**       test_markers_out,
    const Vector_u32*  labels,
    const Matrix_i32*  markers,
    uint32_t           i
)
{
    create_vector_u32(train_labels_out, labels->num_elts - 1);
    create_matrix_i32(train_markers_out, markers->num_rows - 1, 
            markers->num_cols);
    create_matrix_i32(test_markers_out, 1, markers->num_cols);

    if (i > 0)
    {
        copy_vector_section_into_vector_u32(*train_labels_out, 0, labels, 0, i);
        copy_matrix_block_into_matrix_i32(*train_markers_out, 0, 0,
                markers, 0, 0, i, markers->num_cols);
    }
    if (i < labels->num_elts - 1)
    {
        copy_vector_section_into_vector_u32(*train_labels_out, i, 
                labels, i+1, labels->num_elts - 1 - i);
        copy_matrix_block_into_matrix_i32(*train_markers_out, i, 0,
                markers, i+1, 0, markers->num_rows - 1 - i, markers->num_cols);
    }

    copy_matrix_block_into_matrix_i32(*test_markers_out, 0, 0,
            markers, i, 0, 1, markers->num_cols);
}

static void leave_one_out_nb_freq
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
    uint32_t i;

    NB_freq_model_tree* tree  = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.nb_freq_fname)
        return;

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_nb_freq_model_tree(&tree, train_labels, train_markers,
                        opts.nb_freq_fname, opts.nb_freq_dtd_fname)) ||
            (err = predict_labels_with_nb_freq_model_tree(&test_labels, 
                        &test_confs, test_markers, tree, 0)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_nb_freq_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_freq_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_freq_details_cnt_fname, nb_freq_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_freq_confusion_cnt_fname, nb_freq_confusion_pct_fname);

    write_leave_one_out_preds("nb-freq", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, nb_freq_preds_fname);
}

static void leave_one_out_nb_gauss
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
    uint32_t i;

    NB_gauss_model_tree* tree  = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.nb_gauss_fname)
        return;

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_nb_gauss_model_tree(&tree, train_labels, train_markers,
                        opts.nb_gauss_fname, opts.nb_gauss_dtd_fname)) ||
            (err = predict_labels_with_nb_gauss_model_tree(&test_labels, 
                        &test_confs, test_markers, tree, 0)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_nb_gauss_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_gauss_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_gauss_details_cnt_fname, nb_gauss_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_gauss_confusion_cnt_fname, nb_gauss_confusion_pct_fname);

    write_leave_one_out_preds("nb-gauss", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, nb_gauss_preds_fname);
}

static void leave_one_out_nb_gmm
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
    uint32_t i;

    NB_gmm_model_tree* tree  = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.nb_gmm_fname)
        return;

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_nb_gmm_model_tree(&tree, train_labels, train_markers,
                        opts.nb_gmm_fname, opts.nb_gmm_dtd_fname)) ||
            (err = predict_labels_with_nb_gmm_model_tree(&test_labels, 
                        &test_confs, test_markers, tree, 0)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_nb_gmm_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_gmm_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_gmm_details_cnt_fname, nb_gmm_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            nb_gmm_confusion_cnt_fname, nb_gmm_confusion_pct_fname);

    write_leave_one_out_preds("nb-gmm", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, nb_gmm_preds_fname);
}

static void leave_one_out_mv_gmm
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
    uint32_t i;

    MV_gmm_model_tree* tree  = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.mv_gmm_fname)
        return;

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_mv_gmm_model_tree(&tree, train_labels, train_markers,
                        opts.mv_gmm_fname, opts.mv_gmm_dtd_fname)) ||
            (err = predict_labels_with_mv_gmm_model_tree(&test_labels, 
                        &test_confs, test_markers, tree, 0)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_mv_gmm_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            mv_gmm_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            mv_gmm_details_cnt_fname, mv_gmm_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            mv_gmm_confusion_cnt_fname, mv_gmm_confusion_pct_fname);

    write_leave_one_out_preds("mv-gmm", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, mv_gmm_preds_fname);
}

static void leave_one_out_svm
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
#ifdef HAPLO_ENABLE_SVM
    uint32_t i;

    SVM_model_tree* tree  = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.svm_fname)
        return;

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_svm_model_tree(&tree, train_labels, train_markers,
                        opts.svm_fname, opts.svm_dtd_fname)) ||
            (err = predict_labels_with_svm_model_tree(&test_labels, 
                        &test_confs, test_markers, tree)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_svm_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL, 
            svm_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            svm_details_cnt_fname, svm_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            svm_confusion_cnt_fname, svm_confusion_pct_fname);

    write_leave_one_out_preds("svm", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, svm_preds_fname);
#else
    return;
#endif
}

static void leave_one_out_j48
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
#ifdef HAPLO_ENABLE_WEKA
    uint32_t i;
    pid_t    pid;
    char     tmp_dir[1024] = {0};
    char     script[4096] = {0};

    Weka_model_tree* tree     = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.weka_j48_fname)
        return;

    pid = getpid();
    snprintf(tmp_dir, 1024, "%s/.haplo_test_leave_one_out_j48_%u", 
            tmp_dirname, pid);
    snprintf(script, 4096, "mkdir -p %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not create tmp files");
    }

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_weka_j48_model_tree(&tree, train_labels, train_markers,
                        opts.weka_j48_fname, opts.weka_dtd_fname,
                        tmp_dir, opts.weka_jar_fname)) ||
            (err = predict_labels_with_weka_j48_model_tree(&test_labels, 
                        &test_confs, test_markers, tree, opts.weka_jar_fname)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    snprintf(script, 4096, "rm -rf %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not remove tmp files");
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_weka_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            weka_j48_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            weka_j48_details_cnt_fname, weka_j48_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            weka_j48_confusion_cnt_fname, weka_j48_confusion_pct_fname);

    write_leave_one_out_preds("j48", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, weka_j48_preds_fname);
#else
    return;
#endif
}

static void leave_one_out_part
(
    Vector_u32**       labels_out,
    Vector_d**         confs_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
#ifdef HAPLO_ENABLE_WEKA
    uint32_t i;
    pid_t    pid;
    char     tmp_dir[1024] = {0};
    char     script[4096] = {0};

    Weka_model_tree* tree     = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_confs    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.weka_part_fname)
        return;

    pid = getpid();
    snprintf(tmp_dir, 1024, "%s/.haplo_test_leave_one_out_part_%u", 
            tmp_dirname, pid);
    snprintf(script, 4096, "mkdir -p %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not create tmp files");
    }

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(confs_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_weka_part_model_tree(&tree, train_labels, 
                        train_markers, opts.weka_part_fname, 
                        opts.weka_dtd_fname, tmp_dir, 
                        opts.weka_jar_fname)) ||
            (err = predict_labels_with_weka_part_model_tree(&test_labels, 
                        &test_confs, test_markers, tree, opts.weka_jar_fname)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*confs_out)->elts[ i ] = test_confs->elts[ 0 ];
    }

    snprintf(script, 4096, "rm -rf %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not remove tmp files");
    }

    free_vector_u32(test_labels);
    free_vector_d(test_confs);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_weka_model_tree(tree);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            weka_part_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            weka_part_details_cnt_fname, weka_part_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *confs_out, NULL,
            weka_part_confusion_cnt_fname, weka_part_confusion_pct_fname);

    write_leave_one_out_preds("part", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out,
            *confs_out, NULL, weka_part_preds_fname);
#else
    return;
#endif
}

static void leave_one_out_nearest
(
    Vector_u32**       labels_out,
    Vector_d**         dists_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
    uint32_t i;

    Nearest_model* model      = NULL;
    Vector_u32* train_labels  = NULL;
    Matrix_i32* train_markers = NULL;
    Vector_u32* test_labels   = NULL;
    Vector_d*   test_dists    = NULL;
    Matrix_i32* test_markers  = NULL;
    Error* err;

    if (!opts.nearest_fname)
        return;

    create_vector_u32(labels_out, data_labels->num_elts);
    create_vector_d(dists_out, data_labels->num_elts);

    for (i = 0; i < data_labels->num_elts; i++)
    {
        create_leave_one_out_train_and_test_data(&train_labels, &train_markers,
                &test_markers, data_labels, data_markers, i);

        if ((err = train_nearest_model(&model, train_labels, train_markers,
                        opts.nearest_fname, opts.nearest_dtd_fname)) ||
            (err = predict_labels_with_nearest_model(&test_labels, &test_dists,
                        test_markers, model)))
        {
            print_error_msg_exit("haplo-test", err->msg);
        }

        assert(test_labels->num_elts == 1);
        (*labels_out)->elts[ i ] = test_labels->elts[ 0 ];
        (*dists_out)->elts[ i ] = test_dists->elts[ 0 ];
    }

    free_vector_u32(test_labels);
    free_vector_d(test_dists);
    free_matrix_i32(test_markers);
    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);
    free_nearest_model(model);

    find_ancestors(ancestor_types_out, ancestor_labels_out, *labels_out,
            data_labels);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *dists_out, NULL,
            nearest_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *dists_out, NULL,
            nearest_details_cnt_fname, nearest_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, *dists_out, NULL,
            nearest_confusion_cnt_fname, nearest_confusion_pct_fname);

    write_leave_one_out_preds("nearest", data_ids, data_labels,
            *ancestor_types_out, *ancestor_labels_out, *labels_out, *dists_out,
            NULL, nearest_preds_fname);
}

static void leave_one_out_tandem_agree
(
    Vector_u32**       types_out,
    Vector_u32**       labels_out,
    Vector_u32**       ancestor_types_out,
    Vector_u32**       ancestor_labels_out,
    const Vector_u32*  nb_freq_labels, 
    const Vector_u32*  nb_gauss_labels, 
    const Vector_u32*  nb_gmm_labels,
    const Vector_u32*  mv_gmm_labels, 
    const Vector_u32*  svm_labels, 
    const Vector_u32*  j48_labels, 
    const Vector_u32*  part_labels, 
    const Vector_u32*  nearest_labels, 
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels
)
{
    find_ancestors_of_sets(ancestor_types_out, ancestor_labels_out, 
            nb_freq_labels, nb_gauss_labels, nb_gmm_labels, mv_gmm_labels, 
            svm_labels, j48_labels, part_labels, nearest_labels);

    copy_vector_u32(types_out, *ancestor_types_out);
    copy_vector_u32(labels_out, *ancestor_labels_out);

    write_leave_one_out_summary(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, NULL, NULL,
            tandem_agree_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, *ancestor_types_out,
            *ancestor_labels_out, *labels_out, NULL, *types_out,
            tandem_agree_details_cnt_fname, tandem_agree_details_pct_fname);
}

static void leave_one_out_tandem
(
    const Vector_u32*  tandem_types, 
    const Vector_u32*  tandem_labels, 
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    Vector_u32*        ancestor_types, 
    Vector_u32*        ancestor_labels
)
{
    find_tandem_ancestors(&ancestor_types, &ancestor_labels, tandem_types,
            tandem_labels, data_labels);

    write_leave_one_out_summary(data_ids, data_labels, ancestor_types,
            ancestor_labels, tandem_labels, NULL, tandem_types,
            tandem_summary_fname);

    write_leave_one_out_details(data_ids, data_labels, ancestor_types,
            ancestor_labels, tandem_labels, NULL, tandem_types,
            tandem_details_cnt_fname, tandem_details_pct_fname);

    write_leave_one_out_confusion(data_ids, data_labels, ancestor_types,
            ancestor_labels, tandem_labels, NULL, tandem_types,
            tandem_confusion_cnt_fname, tandem_confusion_pct_fname);

    write_leave_one_out_preds("tandem", data_ids, data_labels,
            ancestor_types, ancestor_labels, tandem_labels, NULL,
            tandem_types, tandem_preds_fname);
}

static void create_cross_validation_train_and_test_data
(
    Matblock_u8****    train_ids_out,
    Vector_u32****     train_labels_out,
    Matrix_i32****     train_markers_out, 
    Matblock_u8****    test_ids_out,
    Vector_u32****     test_labels_out,
    Matrix_i32****     test_markers_out,
    const Matblock_u8* data_ids, 
    const Vector_u32*  data_labels, 
    const Matrix_i32*  data_markers
)
{
    uint32_t num_samples;
    uint32_t sample;
    uint32_t num_markers;
    uint32_t marker;
    uint32_t num_ids;
    uint32_t id;
    uint32_t iter;
    uint32_t fold;
    uint32_t num_samples_mod_num_folds;
    uint32_t num_samples_per_fold;
    uint32_t num_times_subtract_a_sample;
    uint32_t train_id_sample;
    uint32_t train_label_sample;
    uint32_t train_marker_sample;
    uint32_t test_id_sample;
    uint32_t test_label_sample;
    uint32_t test_marker_sample;
    uint32_t c;
    int32_t  i, j, k;

    Matblock_u8*** train_ids;
    Vector_u32***  train_labels;
    Matrix_i32***  train_markers;
    Matblock_u8*** test_ids;
    Vector_u32***  test_labels;
    Matrix_i32***  test_markers;
    Vector_i32*    fold_members;
    Matblock_u8*   folded_ids;
    Vector_u32*    folded_labels;
    Matrix_i32*    folded_markers;

    num_ids     = (data_ids) ? data_ids->num_rows : 0;
    num_samples = data_markers->num_rows;
    num_markers = data_markers->num_cols;

    assert(*train_ids_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*train_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*train_markers_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*test_ids_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*test_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*test_markers_out = malloc(num_cv_iters*sizeof(void**)));
    for (iter = 0; iter < num_cv_iters; iter++)
    {
        assert((*train_ids_out)[iter] = calloc(num_cv_folds,sizeof(void*)));
        assert((*train_labels_out)[iter] = calloc(num_cv_folds,sizeof(void*)));
        assert((*train_markers_out)[iter] = calloc(num_cv_folds,sizeof(void*)));
        assert((*test_ids_out)[iter] = calloc(num_cv_folds,sizeof(void*)));
        assert((*test_labels_out)[iter] = calloc(num_cv_folds,sizeof(void*)));
        assert((*test_markers_out)[iter] = calloc(num_cv_folds,sizeof(void*)));
    }

    train_ids     = *train_ids_out;
    train_labels  = *train_labels_out;
    train_markers = *train_markers_out;
    test_ids      = *test_ids_out;
    test_labels   = *test_labels_out;
    test_markers  = *test_markers_out;

    fold_members = NULL;

    for (iter = 0; iter < num_cv_iters; iter++)
    {
        num_samples_mod_num_folds = num_samples % num_cv_folds;
        num_samples_per_fold = ceil(num_samples / (double)num_cv_folds);

        num_times_subtract_a_sample = (num_samples_mod_num_folds > 0) ? 
            num_cv_folds - num_samples_mod_num_folds : 0;

        create_init_vector_i32(&fold_members, num_samples, -1);

        for (fold = 0; fold < num_cv_folds; fold++)
        {
            if (num_times_subtract_a_sample > 0)
            {
                i = num_samples_per_fold - 1;
                num_times_subtract_a_sample--;
            }
            else 
            {
                i = num_samples_per_fold;
            }
            j = i;

            while (j > 0)
            {
                j--;
                k = -1;

                while (k < 0)
                {
                    k = floor((rand()/(double)RAND_MAX) * num_samples);
                    if (k == (int32_t)num_samples) 
                    {
                        k--;
                    }

                    if (fold_members->elts[ k ] < 0)
                    {
                        fold_members->elts[ k ] = fold;
                    }
                    else
                    {
                        k = -1;
                    }
                }
            }

            if (data_ids)
            {
                create_zero_matblock_u8(&(test_ids[ iter ][ fold ]), i,
                        num_ids, data_ids->num_cols);
                create_zero_matblock_u8(&(train_ids[ iter ][ fold ]), 
                        num_samples - i, num_ids, data_ids->num_cols);
            }
            create_zero_vector_u32(&(test_labels[ iter ][ fold ]), i);
            create_zero_matrix_i32(&(test_markers[ iter ][ fold ]), i, 
                    num_markers);
            create_zero_vector_u32(&(train_labels[ iter ][ fold ]), 
                    num_samples - i);
            create_zero_matrix_i32(&(train_markers[ iter ][ fold ]), 
                    num_samples - i, num_markers);
        }

        for (fold = 0; fold < num_cv_folds; fold++)
        {
            test_id_sample = 0;
            test_label_sample = 0;
            test_marker_sample = 0;
            train_id_sample = 0;
            train_label_sample = 0;
            train_marker_sample = 0;

            for (sample = 0; sample < num_samples; sample++)
            {
                if (fold_members->elts[ sample ] == (int32_t)fold)
                {
                    if (data_ids)
                    {
                        folded_ids = test_ids[ iter ][ fold ];
                        for (id = 0; id < num_ids; id++)
                        {
                            for (c = 0; c < data_ids->num_cols; c++)
                            {
                                folded_ids->elts[ test_id_sample ][ id ][ c ] =
                                    data_ids->elts[ sample ][ id ][ c ];
                            }
                        }
                        test_id_sample++;
                    }

                    folded_labels = test_labels[ iter ][ fold ];
                    folded_labels->elts[ test_label_sample ] = 
                        data_labels->elts[ sample ];

                    folded_markers = test_markers[ iter ][ fold ];
                    for (marker = 0; marker < num_markers; marker++)
                    {
                        folded_markers->elts[ test_marker_sample ][ marker ] =
                            data_markers->elts[ sample ][ marker ];
                    }

                    test_label_sample++;
                    test_marker_sample++;
                }
                else
                {
                    if (data_ids)
                    {
                        folded_ids = train_ids[ iter ][ fold ];
                        for (id = 0; id < num_ids; id++)
                        {
                            for (c = 0; c < data_ids->num_cols; c++)
                            {
                                folded_ids->elts[ train_id_sample ][ id ][ c ] =
                                    data_ids->elts[ sample ][ id ][ c ];
                            }
                        }
                        train_id_sample++;
                    }

                    folded_labels = train_labels[ iter ][ fold ];
                    folded_labels->elts[ train_label_sample ] = 
                        data_labels->elts[ sample ];

                    folded_markers = train_markers[ iter ][ fold ];
                    for (marker = 0; marker < num_markers; marker++)
                    {
                        folded_markers->elts[ train_marker_sample ][ marker ] =
                            data_markers->elts[ sample ][ marker ];
                    }

                    train_label_sample++;
                    train_marker_sample++;
                }
            }
        }

    }

    free_vector_i32(fold_members);
}

static void free_cross_validation_train_and_test_data
(
    Matblock_u8***    train_ids,
    Vector_u32***     train_labels,
    Matrix_i32***     train_markers, 
    Matblock_u8***    test_ids,
    Vector_u32***     test_labels,
    Matrix_i32***     test_markers
)
{
    uint32_t i, j;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            free_matblock_u8(train_ids[ i ][ j ]);
            free_vector_u32(train_labels[ i ][ j ]);
            free_matrix_i32(train_markers[ i ][ j ]);
            free_matblock_u8(test_ids[ i ][ j ]);
            free_vector_u32(test_labels[ i ][ j ]);
            free_matrix_i32(test_markers[ i ][ j ]);
        }

        free(train_ids[ i ]);
        free(train_labels[ i ]);
        free(train_markers[ i ]);
        free(test_ids[ i ]);
        free(test_labels[ i ]);
        free(test_markers[ i ]);
    }

    free(train_ids);
    free(train_labels);
    free(train_markers);
    free(test_ids);
    free(test_labels);
    free(test_markers);
}

static void write_cross_validation_summary
(
    Vector_u32*** ancestor_types,
    Vector_u32*** ancestor_labels,
    Vector_u32*** pred_labels,
    Vector_d***   pred_confs,
    Vector_u32*** tandem_types,
    const char*   fname
)
{
    uint32_t    n, N;
    uint32_t    i, j;
    float       D;
    FILE*       fp;
    const char* fmt_1;
    const char* fmt_2;
    xmlDoc*     xml_doc  = NULL;
    xmlNode*    xml_root = NULL;
    xmlNode*    xml_node[3];
    xmlNode*    xml_child = NULL;
    char        xml_buf[256];
    Error*      err;

    Matrix_f* counts = NULL;
    Vector_f* means  = NULL;
    Vector_f* vars   = NULL;
    Vector_f* errs   = NULL;

    if ((err = open_output(&fp, &xml_doc, "haplo-test-cv-summary-out",
                    "haplo-test-cv-summary-out.dtd", fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    create_zero_matrix_f(&counts, num_cv_iters*num_cv_folds, 3);

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fmt_1 = "%-21s  %-21s  %-21s\n";
                fmt_2 = "%-8s %-6s %-5s  %-8s %-6s %-5s  %-8s %-6s %-5s\n";
                break;
            case HAPLO_OUTPUT_CSV:
                fmt_1 = "%s,,,%s,,,%s,,\n";
                fmt_2 = "%s,%s,%s,%s,%s,%s,%s,%s,%s\n";
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(fp, fmt_1, "Direct", "Indirect", "None");
        fprintf(fp, fmt_2, "Mean", "Dev", "Err", "Mean", "Dev", "Err", "Mean",
                "Dev", "Err");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            N = ancestor_types[ i ][ j ]->num_elts;

            for (n = 0; n < N; n++)
            {
                if (!tandem_types || 
                        tandem_types[i][j]->elts[n] != HAPLO_ANCESTOR_NONE)
                {
                    switch (ancestor_types[ i ][ j ]->elts[ n ])
                    {
                        case HAPLO_ANCESTOR_DIRECT:
                            counts->elts[ i*num_cv_folds + j ][ 0 ]++;
                            break;
                        case HAPLO_ANCESTOR_INDIRECT:
                            counts->elts[ i*num_cv_folds + j ][ 1 ]++;
                            break;
                        case HAPLO_ANCESTOR_NONE:
                            counts->elts[ i*num_cv_folds + j ][ 2 ]++;
                            break;
                    }
                }
            }
        }
    }

    ind_mv_sample_stats_f(&means, &vars, &errs, counts);

    switch (opts.output_format)
    {
        case HAPLO_OUTPUT_TXT:
            fmt_1 = "%-8.1f %-6.1f %-5.1f  %-8.1f %-6.1f %-5.1f  %-8.1f %-6.1f %-5.1f\n";
            fprintf(fp, fmt_1, means->elts[0], sqrt(vars->elts[0]),
                    errs->elts[0], means->elts[1], sqrt(vars->elts[1]),
                    errs->elts[1], means->elts[2], sqrt(vars->elts[2]),
                    errs->elts[2]);
            break;
        case HAPLO_OUTPUT_CSV:
            fmt_1 = "%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f\n";
            fprintf(fp, fmt_1, means->elts[0], sqrt(vars->elts[0]),
                    errs->elts[0], means->elts[1], sqrt(vars->elts[1]),
                    errs->elts[1], means->elts[2], sqrt(vars->elts[2]),
                    errs->elts[2]);
            break;
        case HAPLO_OUTPUT_XML:
            xml_node[0] = XMLNewChild(xml_root, "direct", NULL);
            xml_node[1] = XMLNewChild(xml_root, "indirect", NULL);
            xml_node[2] = XMLNewChild(xml_root, "none", NULL);
            for (i = 0; i < 3; i++)
            {
                xml_child = XMLNewChild(xml_node[ i ], "count", NULL);
                snprintf(xml_buf, 256, "%.1f", means->elts[ i ]);
                XMLNewChild(xml_child, "mean", xml_buf);
                snprintf(xml_buf, 256, "%.1f", sqrt(vars->elts[ i ]));
                XMLNewChild(xml_child, "dev", xml_buf);
                snprintf(xml_buf, 256, "%.1f", errs->elts[ i ]);
                XMLNewChild(xml_child, "err", xml_buf);
            }
            break;
    }

    D = means->elts[0] + means->elts[1] + means->elts[2];
    if (D > 0)
    {
        multiply_vector_by_scalar_f(&means, means, 1/D);
        multiply_vector_by_scalar_f(&vars, vars, 1/(D*D));
        multiply_vector_by_scalar_f(&errs, errs, 1/D);
    }

    switch (opts.output_format)
    {
        case HAPLO_OUTPUT_TXT:
            fmt_1 = "%-8.3f %-6.3f %-5.3f  %-8.3f %-6.3f %-5.3f  %-8.3f %-6.3f %-5.3f\n";
            fprintf(fp, fmt_1, means->elts[0], sqrt(vars->elts[0]),
                    errs->elts[0], means->elts[1], sqrt(vars->elts[1]),
                    errs->elts[1], means->elts[2], sqrt(vars->elts[2]),
                    errs->elts[2]);
            break;
        case HAPLO_OUTPUT_CSV:
            fmt_1 = "%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n";
            fprintf(fp, fmt_1, means->elts[0], sqrt(vars->elts[0]),
                    errs->elts[0], means->elts[1], sqrt(vars->elts[1]),
                    errs->elts[1], means->elts[2], sqrt(vars->elts[2]),
                    errs->elts[2]);
            break;
        case HAPLO_OUTPUT_XML:
            for (i = 0; i < 3; i++)
            {
                xml_child = XMLNewChild(xml_node[ i ], "percent", NULL);
                snprintf(xml_buf, 256, "%.3f", means->elts[ i ]);
                XMLNewChild(xml_child, "mean", xml_buf);
                snprintf(xml_buf, 256, "%.3f", sqrt(vars->elts[ i ]));
                XMLNewChild(xml_child, "dev", xml_buf);
                snprintf(xml_buf, 256, "%.3f", errs->elts[ i ]);
                XMLNewChild(xml_child, "err", xml_buf);
            }
            break;
    }

    free_matrix_f(counts);
    free_vector_f(means);
    free_vector_f(vars);
    free_vector_f(errs);

    if ((err = close_output(fp, xml_doc, fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void write_cross_validation_details
(
    Vector_u32*** ancestor_types,
    Vector_u32*** ancestor_labels,
    Vector_u32*** pred_labels,
    Vector_d***   pred_confs,
    Vector_u32*** tandem_types,
    const char*   cnt_fname,
    const char*   pct_fname
)
{
    uint32_t    i, j, k;
    uint32_t    n, nn, N;
    float       D;
    const char* label;
    const char* fmt_1;
    const char* fmt_2;
    FILE*       cnt_fp;
    FILE*       pct_fp;
    xmlDoc*     xml_doc    = NULL;
    xmlNode*    xml_root   = NULL;
    xmlNode*    xml_nd1    = NULL;
    xmlNode*    xml_nd2[3] = {0};
    xmlNode*    xml_nd3    = NULL;
    char        xml_buf[256];
    Error*      err;

    Matrix_f* counts[3]   = {0};
    Vector_f* stats[3][3] = {{0}};

    if (!(pred_labels[0][0]))
        return;

    if ((err = open_output(&cnt_fp, &xml_doc, "haplo-test-cv-details-out",
                    "haplo-test-cv-details-out.dtd", cnt_fname)) ||
        (err = open_output(&pct_fp, NULL, NULL, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    N = get_num_haplo_groups();
    for (i = 0; i < 3; i++)
    {
        create_zero_matrix_f(&(counts[ i ]), num_cv_iters*num_cv_folds, N);
    }

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            for (k = 0; k < ancestor_types[ i ][ j ]->num_elts; k++)
            {
                if (!tandem_types || 
                        tandem_types[i][j]->elts[k] != HAPLO_ANCESTOR_NONE)
                {
                    assert(pred_labels[ i ][ j ]->elts[ k ] < N);
                    n = pred_labels[ i ][ j ]->elts[ k ];

                    switch (ancestor_types[ i ][ j ]->elts[ k ])
                    {
                        case HAPLO_ANCESTOR_DIRECT:
                            counts[0]->elts[ i*num_cv_folds + j ][ n ]++;
                            break;
                        case HAPLO_ANCESTOR_INDIRECT:
                            counts[1]->elts[ i*num_cv_folds + j ][ n ]++;
                            break;
                        case HAPLO_ANCESTOR_NONE:
                            counts[2]->elts[ i*num_cv_folds + j ][ n ]++;
                            break;
                    }
                }
            }
        }
    }

    for (i = 0; i < 3; i++)
    {
        ind_mv_sample_stats_f(&(stats[ i ][0]), &(stats[ i ][1]), 
                &(stats[ i ][2]), counts[ i ]);
    }

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fmt_1 = "%-10s  %-21s  %-21s  %-21s\n";
                fmt_2 = "%-10s  %-8s %-6s %-5s  %-8s %-6s %-5s  %-8s %-6s %-5s\n";
                break;
            case HAPLO_OUTPUT_CSV:
                fmt_1 = "%s,%s,,,%s,,,%s,,\n";
                fmt_2 = "%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n";
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(cnt_fp, fmt_1, "Predicted", "Direct", "Indirect", "None");
        fprintf(cnt_fp, fmt_2, "Label", "Mean", "Dev", "Err", "Mean", "Dev",
                "Err", "Mean", "Dev", "Err");
        fprintf(pct_fp, fmt_1, "Predicted", "Direct", "Indirect", "None");
        fprintf(pct_fp, fmt_2, "Label", "Mean", "Dev", "Err", "Mean", "Dev",
                "Err", "Mean", "Dev", "Err");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    nn = 0;
    for (n = 0; n < N; n++)
    {
        D = stats[0][0]->elts[n] + stats[1][0]->elts[n] + stats[2][0]->elts[n];
        if (D > 0)
        {
            lookup_haplo_group_label_from_index(&label, n);

            switch (opts.output_format)
            {
                case HAPLO_OUTPUT_TXT:
                    fmt_1 = "%-10s  %-8.1f %-6.1f %-5.1f  %-8.1f %-6.1f %-5.1f  %-8.1f %-6.1f %-5.1f\n";
                    fprintf(cnt_fp, fmt_1, label, 
  stats[0][0]->elts[n], sqrt(stats[0][1]->elts[n]), stats[0][2]->elts[n],
  stats[1][0]->elts[n], sqrt(stats[1][1]->elts[n]), stats[1][2]->elts[n], 
  stats[2][0]->elts[n], sqrt(stats[2][1]->elts[n]), stats[2][2]->elts[n]);
                    fmt_1 = "%-10s  %-8.3f %-6.3f %-5.3f  %-8.3f %-6.3f %-5.3f  %-8.3f %-6.3f %-5.3f\n";
                    fprintf(pct_fp, fmt_1, label, 
  stats[0][0]->elts[n]/D, sqrt(stats[0][1]->elts[n])/D, stats[0][2]->elts[n]/D,
  stats[1][0]->elts[n]/D, sqrt(stats[1][1]->elts[n])/D, stats[1][2]->elts[n]/D,
  stats[2][0]->elts[n]/D, sqrt(stats[2][1]->elts[n])/D, stats[2][2]->elts[n]/D);
                    break;
                case HAPLO_OUTPUT_CSV:
                    fmt_1 = "%s,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f\n";
                    fprintf(cnt_fp, fmt_1, label, 
  stats[0][0]->elts[n], sqrt(stats[0][1]->elts[n]), stats[0][2]->elts[n],
  stats[1][0]->elts[n], sqrt(stats[1][1]->elts[n]), stats[1][2]->elts[n], 
  stats[2][0]->elts[n], sqrt(stats[2][1]->elts[n]), stats[2][2]->elts[n]);
                    fmt_1 = "%s,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f,%.3f\n";
                    fprintf(pct_fp, fmt_1, label, 
  stats[0][0]->elts[n]/D, sqrt(stats[0][1]->elts[n])/D, stats[0][2]->elts[n]/D,
  stats[1][0]->elts[n]/D, sqrt(stats[1][1]->elts[n])/D, stats[1][2]->elts[n]/D,
  stats[2][0]->elts[n]/D, sqrt(stats[2][1]->elts[n])/D, stats[2][2]->elts[n]/D);
                    break;
                case HAPLO_OUTPUT_XML:
                    xml_nd1 = XMLNewChild(xml_root, "predicted", NULL);
                    snprintf(xml_buf, 256, "%d", nn+1);
                    XMLNewProp(xml_nd1, "number", xml_buf);
                    XMLNewChild(xml_nd1, "label", label);
                    xml_nd2[0] = XMLNewChild(xml_nd1, "direct", 0);
                    xml_nd2[1] = XMLNewChild(xml_nd1, "indirect", 0);
                    xml_nd2[2] = XMLNewChild(xml_nd1, "none", 0);
                    for (i = 0; i < 3; i++)
                    {
                        xml_nd3 = XMLNewChild(xml_nd2[i], "count", NULL);
                        snprintf(xml_buf, 256, "%.1f", stats[i][0]->elts[n]);
                        XMLNewChild(xml_nd3, "mean", xml_buf);
                        snprintf(xml_buf, 256, "%.1f", 
                                sqrt(stats[i][1]->elts[n]));
                        XMLNewChild(xml_nd3, "dev", xml_buf);
                        snprintf(xml_buf, 256, "%.1f", stats[i][2]->elts[n]);
                        XMLNewChild(xml_nd3, "err", xml_buf);

                        xml_nd3 = XMLNewChild(xml_nd2[i], "percent", NULL);
                        snprintf(xml_buf, 256, "%.1f", 
                                stats[i][0]->elts[n]/D);
                        XMLNewChild(xml_nd3, "mean", xml_buf);
                        snprintf(xml_buf, 256, "%.1f", 
                                sqrt(stats[i][1]->elts[n])/D);
                        XMLNewChild(xml_nd3, "dev", xml_buf);
                        snprintf(xml_buf, 256, "%.1f", 
                                stats[i][2]->elts[n]/D);
                        XMLNewChild(xml_nd3, "err", xml_buf);
                    }
                    break;
            }

            nn++;
        }
    }

    for (i = 0; i < 3; i++)
    {
        free_matrix_f(counts[ i ]);

        for (j = 0; j < 3; j++)
        {
            free_vector_f(stats[ i ][ j ]);
        }
    }

    if ((err = close_output(cnt_fp, xml_doc, cnt_fname)) ||
        (err = close_output(pct_fp, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void write_cross_validation_confusion
(
    Vector_u32*** test_labels,
    Vector_u32*** pred_labels,
    Vector_d***   pred_confs,
    Vector_u32*** tandem_types,
    const char*   cnt_fname,
    const char*   pct_fname
)
{
    uint32_t    i, j, k;
    uint32_t    n_1, n_2, N;
    uint32_t    nn_1, nn_2;
    float       D;
    const char* label;
    FILE*       cnt_fp;
    FILE*       pct_fp;
    xmlDoc*     xml_doc    = NULL;
    xmlNode*    xml_root   = NULL;
    xmlNode*    xml_actual = NULL;
    xmlNode*    xml_pred   = NULL;
    char        xml_buf[256];
    Error*      err;

    Vector_f* actual_counts    = NULL;
    Vector_f* predicted_counts = NULL;
    Matrix_f* confusion        = NULL;
    Matrix_f* confusion_pct    = NULL;

    if (!(pred_labels[0][0]))
        return;

    if ((err = open_output(&cnt_fp, &xml_doc, "haplo-test-cv-confusion-out",
                    "haplo-test-cv-confusion-out.dtd", cnt_fname)) ||
        (err = open_output(&pct_fp, NULL, NULL, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    N = get_num_haplo_groups();
    create_zero_vector_f(&actual_counts, N);
    create_zero_vector_f(&predicted_counts, N);
    create_zero_matrix_f(&confusion, N, N);
    create_zero_matrix_f(&confusion_pct, N, N);

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            for (k = 0; k < test_labels[ i ][ j ]->num_elts; k++)
            {
                if (!tandem_types || 
                        tandem_types[i][j]->elts[k] != HAPLO_ANCESTOR_NONE)
                {
                assert(test_labels[ i ][ j ]->elts[ k ] < N);
                assert(pred_labels[ i ][ j ]->elts[ k ] < N);

                actual_counts->elts[ test_labels[ i ][ j ]->elts[ k ] ]++;
                predicted_counts->elts[ pred_labels[ i ][ j ]->elts[ k ] ]++;
                confusion->elts[ test_labels[ i ][ j ]->elts[ k ] ]
                               [ pred_labels[ i ][ j ]->elts[ k ] ]++;
                }
            }
        }
    }

    multiply_vector_by_scalar_f(&actual_counts, actual_counts,
            1.0f/(num_cv_iters*num_cv_folds));
    multiply_vector_by_scalar_f(&predicted_counts, predicted_counts,
            1.0f/(num_cv_iters*num_cv_folds));
    multiply_matrix_by_scalar_f(&confusion, confusion,
            1.0f/(num_cv_iters*num_cv_folds));

    copy_matrix_f(&confusion_pct, confusion);
    for (n_2 = 0; n_2 < N; n_2++)
    {
        D = predicted_counts->elts[ n_2 ];
        if (D > 0)
        {
            for (n_1 = 0; n_1 < N; n_1++)
            {
                confusion_pct->elts[ n_1 ][ n_2 ] /= D;
            }
        }
    }

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fprintf(cnt_fp, "%-10s", "Actual");
                fprintf(pct_fp, "%-10s", "Actual");
                for (n_2 = 0; n_2 < N; n_2++)
                {
                    if (predicted_counts->elts[ n_2 ])
                    {
                        lookup_haplo_group_label_from_index(&label, n_2);
                        fprintf(cnt_fp, "  %-10s", label);
                        fprintf(pct_fp, "  %-10s", label);
                    }
                }
                break;
            case HAPLO_OUTPUT_CSV:
                fprintf(cnt_fp, "%s", "Actual");
                fprintf(pct_fp, "%s", "Actual");
                for (n_2 = 0; n_2 < N; n_2++)
                {
                    if (predicted_counts->elts[ n_2 ])
                    {
                        lookup_haplo_group_label_from_index(&label, n_2);
                        fprintf(cnt_fp, ",%s", label);
                        fprintf(pct_fp, ",%s", label);
                    }
                }
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(cnt_fp, "\n");
        fprintf(pct_fp, "\n");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    nn_1 = 0;
    for (n_1 = 0; n_1 < N; n_1++)
    {
        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                if (actual_counts->elts[ n_1 ] > 0)
                {
                    lookup_haplo_group_label_from_index(&label, n_1);
                    fprintf(cnt_fp, "%-10s", label);
                    fprintf(pct_fp, "%-10s", label);

                    nn_2 = 0;
                    for (n_2 = 0; n_2 < N; n_2++)
                    {
                        if (predicted_counts->elts[ n_2 ] > 0)
                        {
                            fprintf(cnt_fp, "  %-10.1f", 
                                    confusion->elts[ n_1 ][ n_2 ]);
                            fprintf(pct_fp, "  %-10.3f", 
                                    confusion_pct->elts[ n_1 ][ n_2 ]);
                            nn_2++;
                        }
                    }
                    fprintf(cnt_fp, "\n");
                    fprintf(pct_fp, "\n");
                    nn_1++;
                }
                break;
            case HAPLO_OUTPUT_CSV:
                if (actual_counts->elts[ n_1 ] > 0)
                {
                    lookup_haplo_group_label_from_index(&label, n_1);
                    fprintf(cnt_fp, "%s", label);
                    fprintf(pct_fp, "%s", label);

                    nn_2 = 0;
                    for (n_2 = 0; n_2 < N; n_2++)
                    {
                        if (predicted_counts->elts[ n_2 ] > 0)
                        {
                            fprintf(cnt_fp, ",%.1f", 
                                    confusion->elts[ n_1 ][ n_2 ]);
                            fprintf(pct_fp, ",%.3f", 
                                    confusion_pct->elts[ n_1 ][ n_2 ]);
                            nn_2++;
                        }
                    }
                    fprintf(cnt_fp, "\n");
                    fprintf(pct_fp, "\n");
                    nn_1++;
                }
                break;
            case HAPLO_OUTPUT_XML:
                if (actual_counts->elts[ n_1 ] > 0)
                {
                    lookup_haplo_group_label_from_index(&label, n_1);
                    xml_actual = XMLNewChild(xml_root, "actual", NULL);
                    snprintf(xml_buf, 256, "%d", nn_1+1);
                    XMLNewProp(xml_actual, "number", xml_buf);
                    XMLNewChild(xml_actual, "label", label);

                    nn_2 = 0;
                    for (n_2 = 0; n_2 < N; n_2++)
                    {
                        if (predicted_counts->elts[ n_2 ] > 0)
                        {
                            lookup_haplo_group_label_from_index(&label, n_2);
                            xml_pred = XMLNewChild(xml_actual, "predicted", 0);
                            snprintf(xml_buf, 256, "%d", nn_2+1);
                            XMLNewProp(xml_pred, "number", xml_buf);
                            XMLNewChild(xml_pred, "label", label);

                            snprintf(xml_buf, 256, "%.1f", 
                                    confusion->elts[n_1][n_2]);
                            XMLNewChild(xml_pred, "count", xml_buf);

                            snprintf(xml_buf, 256, "%.3f", 
                                    confusion_pct->elts[n_1][n_2]);
                            XMLNewChild(xml_pred, "percent", xml_buf);

                            nn_2++;
                        }
                    }
                    nn_1++;
                }
                break;
        }
    }

    free_vector_f(actual_counts);
    free_vector_f(predicted_counts);
    free_matrix_f(confusion);
    free_matrix_f(confusion_pct);

    if ((err = close_output(cnt_fp, xml_doc, cnt_fname)) ||
        (err = close_output(pct_fp, NULL, pct_fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void write_cross_validation_preds
(
    const char*    type,
    Matblock_u8*** data_ids,
    Vector_u32***  data_labels,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels,
    Vector_u32***  pred_labels,
    Vector_d***    pred_confs,
    Vector_u32***  tandem_types,
    const char*    fname
)
{
    uint32_t    i, j, k;
    FILE*       fp;
    xmlDoc*     xml_doc    = NULL;
    xmlNode*    xml_root   = NULL;
    xmlNode*    xml_node   = NULL;
    char        xml_buf[256] = {0};
    Error*      err;

    if (!(pred_labels[0][0]))
        return;

    if ((err = open_output(&fp, &xml_doc, "haplo-test-cv-predictions-out",
                    "haplo-test-cv-predictions-out.dtd", fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }

    if (opts.header_out && opts.output_format != HAPLO_OUTPUT_XML)
    {
        if (data_ids)
        {
            for (j = 0; j < data_ids[0][0]->num_rows; j++)
            {
                switch (opts.output_format)
                {
                    case HAPLO_OUTPUT_TXT:
                        fprintf(fp, "ID %-7d  ", j+1);
                        break;
                    case HAPLO_OUTPUT_CSV:
                        fprintf(fp, "ID %d,", j+1);
                        break;
                    case HAPLO_OUTPUT_XML:
                        break;
                }
            }
        }

        switch (opts.output_format)
        {
            case HAPLO_OUTPUT_TXT:
                fprintf(fp, "%-10s  %-10s  %-4s", "Actual", "Ancestor", "Type");
                if (pred_labels) 
                    fprintf(fp, "  %-10s  %-5s", "Prediction", "Conf");
                break;
            case HAPLO_OUTPUT_CSV:
                fprintf(fp, "%s,%s,%s", "Actual", "Ancestor", "Type");
                if (pred_labels) 
                    fprintf(fp, ",%s,%s", "Prediction", "Conf");
                break;
            case HAPLO_OUTPUT_XML:
                break;
        }
        fprintf(fp, "\n");
    }
    else if (opts.output_format == HAPLO_OUTPUT_XML)
    {
        xml_root = xmlDocGetRootElement(xml_doc);
    }

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            for (k = 0; k < data_labels[i][j]->num_elts; k++)
            {
                if (!tandem_types || 
                        tandem_types[i][j]->elts[k] != HAPLO_ANCESTOR_NONE)
                {
                if (opts.output_format == HAPLO_OUTPUT_XML)
                {
                    xml_node = XMLNewChild(xml_root, "sample", NULL);
                    snprintf(xml_buf, 256, "%d", k+1);
                    XMLNewProp(xml_node, "number", xml_buf);
                }

                write_ids(data_ids[i][j], k, HAPLO_SEP_SUFFIX, fp, xml_node);
                write_label(data_labels[i][j], k, HAPLO_SEP_SUFFIX, fp, 
                        xml_node);

                write_ancestor_label(ancestor_types[i][j], 
                        ancestor_labels[i][j], k, HAPLO_SEP_NONE, fp, xml_node);

                write_prediction(type, pred_labels[i][j], (pred_confs) ?
                        pred_confs[i][j] : NULL, k, HAPLO_SEP_PREFIX, fp,
                        xml_node);

                if (opts.output_format != HAPLO_OUTPUT_XML)
                {
                    fprintf(fp, "\n");
                }
                }
            }
        }
    }

    if ((err = close_output(fp, xml_doc, fname)))
    {
        print_error_msg("haplo-test", err->msg);
    }
}

static void cross_validate_nb_freq
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    NB_freq_model_tree* tree = NULL;
    Error* err;

    if (!opts.nb_freq_fname)
        return;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_nb_freq_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.nb_freq_fname, 
                            opts.nb_freq_dtd_fname)) ||
                (err = predict_labels_with_nb_freq_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], 
                            tree, 0)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    free_nb_freq_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, nb_freq_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, nb_freq_details_cnt_fname,
            nb_freq_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, nb_freq_confusion_cnt_fname, nb_freq_confusion_pct_fname);

    write_cross_validation_preds("nb-freq", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, nb_freq_preds_fname);
}

static void cross_validate_nb_gauss
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    NB_gauss_model_tree* tree = NULL;
    Error* err;

    if (!opts.nb_gauss_fname)
        return;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_nb_gauss_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.nb_gauss_fname, 
                            opts.nb_gauss_dtd_fname)) ||
                (err = predict_labels_with_nb_gauss_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], 
                            tree, 0)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    free_nb_gauss_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, nb_gauss_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, nb_gauss_details_cnt_fname,
            nb_gauss_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, nb_gauss_confusion_cnt_fname, nb_gauss_confusion_pct_fname);

    write_cross_validation_preds("nb-gauss", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, nb_gauss_preds_fname);
}

static void cross_validate_nb_gmm
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    NB_gmm_model_tree* tree = NULL;
    Error* err;

    if (!opts.nb_gmm_fname)
        return;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_nb_gmm_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.nb_gmm_fname, 
                            opts.nb_gmm_dtd_fname)) ||
                (err = predict_labels_with_nb_gmm_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], 
                            tree, 0)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    free_nb_gmm_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, nb_gmm_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, nb_gmm_details_cnt_fname,
            nb_gmm_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, nb_gmm_confusion_cnt_fname, nb_gmm_confusion_pct_fname);

    write_cross_validation_preds("nb-gmm", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, nb_gmm_preds_fname);
}

static void cross_validate_mv_gmm
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    MV_gmm_model_tree* tree = NULL;
    Error* err;

    if (!opts.mv_gmm_fname)
        return;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_mv_gmm_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.mv_gmm_fname, 
                            opts.mv_gmm_dtd_fname)) ||
                (err = predict_labels_with_mv_gmm_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], 
                            tree, 0)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    free_mv_gmm_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, mv_gmm_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, mv_gmm_details_cnt_fname,
            mv_gmm_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, mv_gmm_confusion_cnt_fname, mv_gmm_confusion_pct_fname);

    write_cross_validation_preds("mv-gmm", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, mv_gmm_preds_fname);
}

static void cross_validate_svm
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
#ifdef HAPLO_ENABLE_SVM
    uint32_t i, j;

    SVM_model_tree* tree = NULL;
    Error* err;

    if (!opts.svm_fname)
        return;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_svm_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.svm_fname, 
                            opts.svm_dtd_fname)) ||
                (err = predict_labels_with_svm_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], tree)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    free_svm_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, svm_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, svm_details_cnt_fname,
            svm_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, svm_confusion_cnt_fname, svm_confusion_pct_fname);

    write_cross_validation_preds("svm", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, svm_preds_fname);
#else
    return;
#endif
}

static void cross_validate_j48
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
#ifdef HAPLO_ENABLE_WEKA
    uint32_t i, j;
    pid_t    pid;
    char     tmp_dir[1024] = {0};
    char     script[4096] = {0};

    Weka_model_tree* tree = NULL;
    Error* err;

    if (!opts.weka_j48_fname)
        return;

    pid = getpid();
    snprintf(tmp_dir, 1024, "%s/.haplo_test_cross_validate_j48_%u", 
            tmp_dirname, pid);
    snprintf(script, 4096, "mkdir -p %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not create tmp files");
    }

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_weka_j48_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.weka_j48_fname, 
                            opts.weka_dtd_fname, tmp_dir,
                            opts.weka_jar_fname)) ||
                (err = predict_labels_with_weka_j48_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], 
                            tree, opts.weka_jar_fname)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    snprintf(script, 4096, "rm -rf %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not remove tmp files");
    }

    free_weka_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, weka_j48_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, weka_j48_details_cnt_fname,
            weka_j48_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, weka_j48_confusion_cnt_fname, weka_j48_confusion_pct_fname);

    write_cross_validation_preds("j48", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, weka_j48_preds_fname);
#else
    return;
#endif
}

static void cross_validate_part
(
    Vector_u32**** labels_out,
    Vector_d****   confs_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
#ifdef HAPLO_ENABLE_WEKA
    uint32_t i, j;
    pid_t    pid;
    char     tmp_dir[1024] = {0};
    char     script[4096] = {0};

    Weka_model_tree* tree = NULL;
    Error* err;

    if (!opts.weka_part_fname)
        return;

    pid = getpid();
    snprintf(tmp_dir, 1024, "%s/.haplo_test_cross_validate_part_%u", 
            tmp_dirname, pid);
    snprintf(script, 4096, "mkdir -p %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not create tmp files");
    }

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_weka_part_model_tree(&tree, train_labels[i][j], 
                            train_markers[i][j], opts.weka_part_fname, 
                            opts.weka_dtd_fname, tmp_dir,
                            opts.weka_jar_fname)) ||
                (err = predict_labels_with_weka_part_model_tree(
                            &((*labels_out)[i][j]), 
                            &((*confs_out)[i][j]), test_markers[i][j], 
                            tree, opts.weka_jar_fname)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    snprintf(script, 4096, "rm -rf %s", tmp_dir);
    if (system(script) == 127)
    {
        print_error_msg_exit("haplo-test", "Could not remove tmp files");
    }

    free_weka_model_tree(tree);

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, *confs_out, NULL, weka_part_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, 
            *labels_out, *confs_out, NULL, weka_part_details_cnt_fname,
            weka_part_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *confs_out,
            NULL, weka_part_confusion_cnt_fname, weka_part_confusion_pct_fname);

    write_cross_validation_preds("part", test_ids, test_labels, 
            ancestor_types, ancestor_labels, *labels_out, *confs_out, 
            NULL, weka_part_preds_fname);
#else
    return;
#endif
}

static void cross_validate_nearest
(
    Vector_u32**** labels_out,
    Vector_d****   dists_out,
    Matblock_u8*** train_ids,
    Vector_u32***  train_labels,
    Matrix_i32***  train_markers,
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Matrix_i32***  test_markers,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    Nearest_model* model = NULL;
    Error* err;

    if (!opts.nearest_fname)
        return;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            if ((err = train_nearest_model(&model, train_labels[i][j], 
                            train_markers[i][j], opts.nearest_fname, 
                            opts.nearest_dtd_fname)) ||
                (err = predict_labels_with_nearest_model(
                            &((*labels_out)[i][j]), 
                            &((*dists_out)[i][j]), test_markers[i][j], model)))
            {
                print_error_msg_exit("haplo-test", err->msg);
            }

            find_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), (*labels_out)[i][j], 
                    test_labels[i][j]);
        }
    }

    free_nearest_model(model);

    write_cross_validation_summary(ancestor_types, ancestor_labels, *labels_out,
            *dists_out, NULL, nearest_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels, *labels_out,
            *dists_out, NULL, nearest_details_cnt_fname,
            nearest_details_pct_fname);

    write_cross_validation_confusion(test_labels, *labels_out, *dists_out,
            NULL, nearest_confusion_cnt_fname, nearest_confusion_pct_fname);

    write_cross_validation_preds("nearest", test_ids, test_labels,
            ancestor_types, ancestor_labels, *labels_out, *dists_out,
            NULL, nearest_preds_fname);
}

static void cross_validate_tandem_agree
(
    Vector_u32**** types_out, 
    Vector_u32**** labels_out,
    Vector_u32***  nb_freq_labels, 
    Vector_u32***  nb_gauss_labels, 
    Vector_u32***  nb_gmm_labels,
    Vector_u32***  mv_gmm_labels, 
    Vector_u32***  svm_labels, 
    Vector_u32***  j48_labels, 
    Vector_u32***  part_labels, 
    Vector_u32***  nearest_labels, 
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Vector_u32***  ancestor_types, 
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;
    const Vector_u32*  nb_freq;
    const Vector_u32*  nb_gauss;
    const Vector_u32*  nb_gmm;
    const Vector_u32*  mv_gmm;
    const Vector_u32*  svm;
    const Vector_u32*  j48;
    const Vector_u32*  part;
    const Vector_u32*  nearest;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            nb_freq  = (opts.nb_freq_fname) ? nb_freq_labels[i][j] : NULL;
            nb_gauss = (opts.nb_gauss_fname) ? nb_gauss_labels[i][j] : NULL;
            nb_gmm   = (opts.nb_gmm_fname) ? nb_gmm_labels[i][j] : NULL;
            mv_gmm   = (opts.mv_gmm_fname) ? mv_gmm_labels[i][j] : NULL;
#ifdef HAPLO_ENABLE_SVM
            svm      = (opts.svm_fname) ? svm_labels[i][j] : NULL;
#else    
            svm      = NULL;
#endif
#ifdef HAPLO_ENABLE_WEKA
            j48      = (opts.weka_j48_fname) ? j48_labels[i][j] : NULL;
            part     = (opts.weka_part_fname) ? part_labels[i][j] : NULL;
#else
            j48      = NULL;
            part     = NULL;
#endif
            nearest  = (opts.nearest_fname) ? nearest_labels[i][j] : NULL;

            find_ancestors_of_sets(&(ancestor_types[i][j]),
                    &(ancestor_labels[i][j]), nb_freq, nb_gauss, nb_gmm,
                    mv_gmm, svm, j48, part, nearest);

            copy_vector_u32(&((*types_out)[i][j]), ancestor_types[i][j]);
            copy_vector_u32(&((*labels_out)[i][j]), ancestor_labels[i][j]);
        }
    }

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            *labels_out, NULL, NULL, tandem_agree_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels,
            *labels_out, NULL, *types_out, tandem_agree_details_cnt_fname,
            tandem_agree_details_pct_fname);
}

static void cross_validate_tandem
(
    Vector_u32***  tandem_types, 
    Vector_u32***  tandem_labels, 
    Matblock_u8*** test_ids,
    Vector_u32***  test_labels,
    Vector_u32***  ancestor_types, 
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j = 0; j < num_cv_folds; j++)
        {
            find_tandem_ancestors(&(ancestor_types[i][j]), 
                    &(ancestor_labels[i][j]), tandem_types[i][j],
                    tandem_labels[i][j], test_labels[i][j]);
        }
    }

    write_cross_validation_summary(ancestor_types, ancestor_labels,
            tandem_labels, NULL, tandem_types, tandem_summary_fname);

    write_cross_validation_details(ancestor_types, ancestor_labels,
            tandem_labels, NULL, tandem_types, tandem_details_cnt_fname,
            tandem_details_pct_fname);

    write_cross_validation_confusion(test_labels, tandem_labels, NULL,
            tandem_types, tandem_confusion_cnt_fname,
            tandem_confusion_pct_fname);

    write_cross_validation_preds("tandem", test_ids, test_labels,
            ancestor_types, ancestor_labels, tandem_labels, NULL,
            tandem_types, tandem_preds_fname);
}

static void leave_one_out
(
    const Matblock_u8* ids,
    const Vector_u32*  labels,
    const Matrix_i32*  markers
)
{
    Vector_u32*  nb_freq_labels  = NULL;
    Vector_d*    nb_freq_confs   = NULL;
    Vector_u32*  nb_gauss_labels = NULL;
    Vector_d*    nb_gauss_confs  = NULL;
    Vector_u32*  nb_gmm_labels   = NULL;
    Vector_d*    nb_gmm_confs    = NULL;
    Vector_u32*  mv_gmm_labels   = NULL;
    Vector_d*    mv_gmm_confs    = NULL;
    Vector_u32*  svm_labels      = NULL;
    Vector_d*    svm_confs       = NULL;
    Vector_u32*  j48_labels      = NULL;
    Vector_d*    j48_confs       = NULL;
    Vector_u32*  part_labels     = NULL;
    Vector_d*    part_confs      = NULL;
    Vector_u32*  nearest_labels  = NULL;
    Vector_d*    nearest_dists   = NULL;
    Vector_u32*  tandem_types    = NULL;
    Vector_u32*  tandem_labels   = NULL;
    Vector_u32*  ancestor_types  = NULL;
    Vector_u32*  ancestor_labels = NULL;

    leave_one_out_nb_freq(&nb_freq_labels, &nb_freq_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_nb_gauss(&nb_gauss_labels, &nb_gauss_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_nb_gmm(&nb_gmm_labels, &nb_gmm_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_mv_gmm(&mv_gmm_labels, &mv_gmm_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_svm(&svm_labels, &svm_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_j48(&j48_labels, &j48_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_part(&part_labels, &part_confs, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    leave_one_out_nearest(&nearest_labels, &nearest_dists, &ancestor_types,
            &ancestor_labels, ids, labels, markers);

    if (tandem)
    {
        leave_one_out_tandem_agree(&tandem_types, &tandem_labels,
                &ancestor_types, &ancestor_labels, nb_freq_labels,
                nb_gauss_labels, nb_gmm_labels, mv_gmm_labels, svm_labels,
                j48_labels, part_labels, nearest_labels, ids, labels);

        leave_one_out_tandem(tandem_types, tandem_labels, ids, labels,
                ancestor_types, ancestor_labels);
    }

    free_vector_u32(nb_freq_labels);
    free_vector_d(nb_freq_confs);
    free_vector_u32(nb_gauss_labels);
    free_vector_d(nb_gauss_confs);
    free_vector_u32(nb_gmm_labels);
    free_vector_d(nb_gmm_confs);
    free_vector_u32(mv_gmm_labels);
    free_vector_d(mv_gmm_confs);
    free_vector_u32(svm_labels);
    free_vector_d(svm_confs);
    free_vector_u32(j48_labels);
    free_vector_d(j48_confs);
    free_vector_u32(part_labels);
    free_vector_d(part_confs);
    free_vector_u32(nearest_labels);
    free_vector_d(nearest_dists);
    free_vector_u32(tandem_labels);
    free_vector_u32(ancestor_types);
    free_vector_u32(ancestor_labels);
}

static void allocate_cross_validation_results
(
    Vector_u32****  nb_freq_labels_out,
    Vector_d****    nb_freq_confs_out,
    Vector_u32****  nb_gauss_labels_out,
    Vector_d****    nb_gauss_confs_out,
    Vector_u32****  nb_gmm_labels_out,
    Vector_d****    nb_gmm_confs_out,
    Vector_u32****  mv_gmm_labels_out,
    Vector_d****    mv_gmm_confs_out,
    Vector_u32****  svm_labels_out,
    Vector_d****    svm_confs_out,
    Vector_u32****  j48_labels_out,
    Vector_d****    j48_confs_out,
    Vector_u32****  part_labels_out,
    Vector_d****    part_confs_out,
    Vector_u32****  nearest_labels_out,
    Vector_d****    nearest_dists_out,
    Vector_u32****  tandem_types_out,
    Vector_u32****  tandem_labels_out,
    Vector_u32****  ancestor_types_out,
    Vector_u32****  ancestor_labels_out,
    uint32_t        num_samples
)
{
    uint32_t i, j;

    assert(*nb_freq_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nb_freq_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nb_gauss_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nb_gauss_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nb_gmm_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nb_gmm_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*mv_gmm_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*mv_gmm_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*svm_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*svm_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*j48_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*j48_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*part_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*part_confs_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nearest_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*nearest_dists_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*tandem_types_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*tandem_labels_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*ancestor_types_out = malloc(num_cv_iters*sizeof(void**)));
    assert(*ancestor_labels_out = malloc(num_cv_iters*sizeof(void**)));

    for (i = 0; i < num_cv_iters; i++)
    {
        assert((*nb_freq_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nb_freq_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nb_gauss_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nb_gauss_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nb_gmm_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nb_gmm_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*mv_gmm_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*mv_gmm_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*svm_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*svm_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*j48_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*j48_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*part_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*part_confs_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nearest_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*nearest_dists_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*tandem_types_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*tandem_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*ancestor_types_out)[i] = calloc(num_cv_folds,sizeof(void*)));
        assert((*ancestor_labels_out)[i] = calloc(num_cv_folds,sizeof(void*)));

        for (j = 0; j < num_cv_folds; j++)
        {
            create_zero_vector_u32(&((*nb_freq_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*nb_freq_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*nb_gauss_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*nb_gauss_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*nb_gmm_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*nb_gmm_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*mv_gmm_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*mv_gmm_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*svm_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*svm_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*j48_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*j48_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*part_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*part_confs_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*nearest_labels_out)[i][j]), num_samples);
            create_zero_vector_d(&((*nearest_dists_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*tandem_types_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*tandem_labels_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*ancestor_types_out)[i][j]), num_samples);
            create_zero_vector_u32(&((*ancestor_labels_out)[i][j]),num_samples);
        }
    }
}

static void free_cross_validation_results
(
    Vector_u32***  nb_freq_labels,
    Vector_d***    nb_freq_confs,
    Vector_u32***  nb_gauss_labels,
    Vector_d***    nb_gauss_confs,
    Vector_u32***  nb_gmm_labels,
    Vector_d***    nb_gmm_confs,
    Vector_u32***  mv_gmm_labels,
    Vector_d***    mv_gmm_confs,
    Vector_u32***  svm_labels,
    Vector_d***    svm_confs,
    Vector_u32***  j48_labels,
    Vector_d***    j48_confs,
    Vector_u32***  part_labels,
    Vector_d***    part_confs,
    Vector_u32***  nearest_labels,
    Vector_d***    nearest_dists,
    Vector_u32***  tandem_types,
    Vector_u32***  tandem_labels,
    Vector_u32***  ancestor_types,
    Vector_u32***  ancestor_labels
)
{
    uint32_t i, j;

    for (i = 0; i < num_cv_iters; i++)
    {
        for (j =0; j < num_cv_folds; j++)
        {
            free_vector_u32(nb_freq_labels[ i ][ j ]);
            free_vector_d(nb_freq_confs[ i ][ j ]);
            free_vector_u32(nb_gauss_labels[ i ][ j ]);
            free_vector_d(nb_gauss_confs[ i ][ j ]);
            free_vector_u32(nb_gmm_labels[ i ][ j ]);
            free_vector_d(nb_gmm_confs[ i ][ j ]);
            free_vector_u32(mv_gmm_labels[ i ][ j ]);
            free_vector_d(mv_gmm_confs[ i ][ j ]);
            free_vector_u32(svm_labels[ i ][ j ]);
            free_vector_d(svm_confs[ i ][ j ]);
            free_vector_u32(j48_labels[ i ][ j ]);
            free_vector_d(j48_confs[ i ][ j ]);
            free_vector_u32(part_labels[ i ][ j ]);
            free_vector_d(part_confs[ i ][ j ]);
            free_vector_u32(nearest_labels[ i ][ j ]);
            free_vector_d(nearest_dists[ i ][ j ]);
            free_vector_u32(tandem_types[ i ][ j ]);
            free_vector_u32(tandem_labels[ i ][ j ]);
            free_vector_u32(ancestor_labels[ i ][ j ]);
            free_vector_u32(ancestor_types[ i ][ j ]);
        }

        free(nb_freq_labels[ i ]); 
        free(nb_freq_confs[ i ]);
        free(nb_gauss_labels[ i ]); 
        free(nb_gauss_confs[ i ]);
        free(nb_gmm_labels[ i ]); 
        free(nb_gmm_confs[ i ]);
        free(mv_gmm_labels[ i ]); 
        free(mv_gmm_confs[ i ]);
        free(svm_labels[ i ]); 
        free(svm_confs[ i ]);
        free(j48_labels[ i ]); 
        free(j48_confs[ i ]);
        free(part_labels[ i ]); 
        free(part_confs[ i ]);
        free(nearest_labels[ i ]);
        free(nearest_dists[ i ]);
        free(tandem_types[ i ]);
        free(tandem_labels[ i ]);
        free(ancestor_labels[ i ]);
        free(ancestor_types[ i ]);
    }

    free(nb_freq_labels); 
    free(nb_freq_confs);
    free(nb_gauss_labels); 
    free(nb_gauss_confs);
    free(nb_gmm_labels); 
    free(nb_gmm_confs);
    free(mv_gmm_labels); 
    free(mv_gmm_confs);
    free(svm_labels); 
    free(svm_confs);
    free(j48_labels); 
    free(j48_confs);
    free(part_labels); 
    free(part_confs);
    free(nearest_labels);
    free(nearest_dists);
    free(tandem_types);
    free(tandem_labels);
    free(ancestor_labels);
    free(ancestor_types);
}

static void cross_validate
(
    const Matblock_u8* data_ids,
    const Vector_u32*  data_labels,
    const Matrix_i32*  data_markers
)
{
    Matblock_u8*** train_ids       = NULL;
    Vector_u32***  train_labels    = NULL;
    Matrix_i32***  train_markers   = NULL;
    Matblock_u8*** test_ids        = NULL;
    Vector_u32***  test_labels     = NULL;
    Matrix_i32***  test_markers    = NULL;
    Vector_u32***  nb_freq_labels  = NULL;
    Vector_d***    nb_freq_confs   = NULL;
    Vector_u32***  nb_gauss_labels = NULL;
    Vector_d***    nb_gauss_confs  = NULL;
    Vector_u32***  nb_gmm_labels   = NULL;
    Vector_d***    nb_gmm_confs    = NULL;
    Vector_u32***  mv_gmm_labels   = NULL;
    Vector_d***    mv_gmm_confs    = NULL;
    Vector_u32***  svm_labels      = NULL;
    Vector_d***    svm_confs       = NULL;
    Vector_u32***  j48_labels      = NULL;
    Vector_d***    j48_confs       = NULL;
    Vector_u32***  part_labels     = NULL;
    Vector_d***    part_confs      = NULL;
    Vector_u32***  nearest_labels  = NULL;
    Vector_d***    nearest_dists   = NULL;
    Vector_u32***  tandem_types    = NULL;
    Vector_u32***  tandem_labels   = NULL;
    Vector_u32***  ancestor_types  = NULL;
    Vector_u32***  ancestor_labels = NULL;

    create_cross_validation_train_and_test_data(&train_ids, &train_labels,
            &train_markers, &test_ids, &test_labels, &test_markers, data_ids,
            data_labels, data_markers);

    allocate_cross_validation_results(&nb_freq_labels, &nb_freq_confs,
            &nb_gauss_labels, &nb_gauss_confs, &nb_gmm_labels, &nb_gmm_confs,
            &mv_gmm_labels, &mv_gmm_confs, &svm_labels, &svm_confs, &j48_labels,
            &j48_confs, &part_labels, &part_confs, &nearest_labels,
            &nearest_dists, &tandem_types, &tandem_labels, &ancestor_types,
            &ancestor_labels, data_labels->num_elts);

    cross_validate_nb_freq(&nb_freq_labels, &nb_freq_confs, train_ids,
            train_labels, train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_nb_gauss(&nb_gauss_labels, &nb_gauss_confs, train_ids,
            train_labels, train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_nb_gmm(&nb_gmm_labels, &nb_gmm_confs, train_ids,
            train_labels, train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_mv_gmm(&mv_gmm_labels, &mv_gmm_confs, train_ids,
            train_labels, train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_svm(&svm_labels, &svm_confs, train_ids, train_labels,
            train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_j48(&j48_labels, &j48_confs, train_ids, train_labels,
            train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_part(&part_labels, &part_confs, train_ids, train_labels,
            train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    cross_validate_nearest(&nearest_labels, &nearest_dists, train_ids,
            train_labels, train_markers, test_ids, test_labels, test_markers,
            ancestor_types, ancestor_labels);

    if (tandem)
    {
        cross_validate_tandem_agree(&tandem_types, &tandem_labels,
                nb_freq_labels, nb_gauss_labels, nb_gmm_labels, mv_gmm_labels,
                svm_labels, j48_labels, part_labels, nearest_labels, test_ids,
                test_labels, ancestor_types, ancestor_labels);

        cross_validate_tandem(tandem_types, tandem_labels, test_ids,
                test_labels, ancestor_types, ancestor_labels);
    }

    free_cross_validation_results(nb_freq_labels, nb_freq_confs,
            nb_gauss_labels, nb_gauss_confs, nb_gmm_labels, nb_gmm_confs,
            mv_gmm_labels, mv_gmm_confs, svm_labels, svm_confs, j48_labels,
            j48_confs, part_labels, part_confs, nearest_labels, nearest_dists, 
            tandem_types, tandem_labels, ancestor_types, ancestor_labels);

    free_cross_validation_train_and_test_data(train_ids, train_labels,
            train_markers, test_ids, test_labels, test_markers);
}


int main(int argc, const char** argv)
{
    int         argi;
    const char* data_fname = "/dev/stdin";
    Error*      err;

    Matblock_u8* ids            = NULL;
    Vector_u32*  labels         = NULL;
    Matrix_i32*  markers        = NULL;

    init_test_options();

    if ((err = process_options(argc, argv, &argi, NUM_OPTS_NO_ARG, opts_no_arg,
                    NUM_OPTS_WITH_ARG, opts_with_arg)) != NULL)
    {
        print_error_msg_exit("haplo-test", err->msg);
    }

    if ((argc - argi) == 1)
    {
        data_fname = argv[ argi ];
    }

    if (num_models_to_test() == 0)
    {
        print_error_msg_exit("haplo-test", "No models to test");
    }

    if ((err = read_haplo_groups(opts.labels_fname)))
    {
        print_error_msg_exit("haplo-test", err->msg);
    }

    if ((err = read_input(&ids, &labels, &markers, data_fname)))
    {
        print_error_msg_exit("haplo-test", err->msg);
    }

    if (!labels)
    {
        print_error_msg("haplo-test", NULL);
        print_error_msg_exit(data_fname, "No labels to test with");
    }

    switch (test_type)
    {
        case HAPLO_TEST_LEAVE_ONE_OUT:
            leave_one_out(ids, labels, markers);
            break;
        case HAPLO_TEST_CROSS_VALIDATE:
            cross_validate(ids, labels, markers);
            break;
    }

    free_matblock_u8(ids);
    free_vector_u32(labels);
    free_matrix_i32(markers);

    if (get_num_unhandled_errors() > 0)
    {
        print_error_msg_exit("haplo-test", "Unhandled errors");
    }

    return EXIT_SUCCESS;
}