weka.c

Go to the documentation of this file.

/*
 * This work is licensed under a Creative Commons 
 * Attribution-Noncommercial-Share Alike 3.0 United States License.
 * 
 *    http://creativecommons.org/licenses/by-nc-sa/3.0/us/
 * 
 * You are free:
 * 
 *    to Share - to copy, distribute, display, and perform the work
 *    to Remix - to make derivative works
 * 
 * Under the following conditions:
 * 
 *    Attribution. You must attribute the work in the manner specified by the
 *    author or licensor (but not in any way that suggests that they endorse you
 *    or your use of the work).
 * 
 *    Noncommercial. You may not use this work for commercial purposes.
 * 
 *    Share Alike. If you alter, transform, or build upon this work, you may
 *    distribute the resulting work only under the same or similar license to
 *    this one.
 * 
 * For any reuse or distribution, you must make clear to others the license
 * terms of this work. The best way to do this is by including this header.
 * 
 * Any of the above conditions can be waived if you get permission from the
 * copyright holder.
 * 
 * Apart from the remix rights granted under this license, nothing in this
 * license impairs or restricts the author's moral rights.
 */


#include <config.h>

#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <inttypes.h>
#include <unistd.h>
#include <string.h>
#include <errno.h>

#include <libxml/tree.h>
#include <libxml/parser.h>
#include <libxml/valid.h>

#ifdef HAPLO_HAVE_DMALLOC
#include <dmalloc.h>
#endif

#include <jwsc/base/error.h>
#include <jwsc/base/file_io.h>
#include <jwsc/vector/vector.h>
#include <jwsc/matrix/matrix.h>

#include "xml.h"
#include "haplo_groups.h"
#include "weka.h"


static Error* read_weka_model_labels
(
    char**    labels_str_out, 
    const char* labels_fname
)
{
    static char buf[4096] = {0};
    char* buf_ptr;
    char  c;
    FILE* fp;
    int   slen = 1024;
    char  str[slen];

    if ((fp = fopen(labels_fname, "r")) == NULL)
    {
        snprintf(str, slen, "%s: %s", labels_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    if (!skip_fp_spaces_and_comments(fp))
    {
        snprintf(str, slen, "%s: %s", labels_fname, 
                "Labels file not formatted properly");
        return JWSC_EARG(str);
    }

    buf_ptr = buf;
    while ((c = fgetc(fp)) != EOF && c != '\n' && buf_ptr != (buf+4096))
    {
        *buf_ptr = c;
        buf_ptr++;
    }
    if (buf_ptr != (buf+4096))
    {
        *buf_ptr = 0;
    }

    if (*labels_str_out)
    {
        free(*labels_str_out);
    }
    *labels_str_out = malloc((strlen(buf)+1)*sizeof(char));
    strcpy(*labels_str_out, buf);

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", labels_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    return NULL;
}


Error* train_weka_j48_model
(
    const Vector_u32* labels, 
    const Matrix_i32* markers,
    const char*       labels_fname,
    const char*       model_fname,
    const char*       weka_jar_fname
)
{
    static char script[4096] = {0};
    static char tmp_data_csv_fname[256] = {0};
    const char* label_str;

    FILE*    fp;
    uint32_t pid;
    uint32_t s, num_samples;
    uint32_t m, num_markers;
    int      slen = 1024;
    char     str[slen];

    pid = getpid();

    snprintf(tmp_data_csv_fname, 255, "/tmp/.weka_j48_train_data_csv-%u", pid);
    if (!(fp = fopen(tmp_data_csv_fname, "w")))
    {
        snprintf(str, slen, "%s: %s", tmp_data_csv_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    num_samples = markers->num_rows;
    num_markers = markers->num_cols;

    for (s = 0; s < num_samples; s++)
    {
        assert(lookup_haplo_group_label_from_index(&label_str, 
                    labels->elts[ s ]) == NULL);
        fprintf(fp, "%s", label_str);
        for (m = 0; m < num_markers; m++)
        {
            fprintf(fp, ",%u", markers->elts[ s ][ m ]);
        }
        fprintf(fp, "\n");
    }

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", tmp_data_csv_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    snprintf(script, 4096, 
            "WEKA_JAR=\"%s\"; \
             JAVA=\"%s\"; \
             EGREP=\"%s\"; \
             AWK=\"%s\"; \
             SED=\"%s\"; \
             TMP_TRAIN_DATA_ARFF=\"/tmp/.weka_j48_train_data_arff-%u\"; \
             TMP_TRAIN_DATA_CSV=\"%s\"; \
             JVM_OPTS=\"-Xmx1024m -cp $WEKA_JAR\"; \
             CSV_CONVERTER=\"weka.core.converters.CSVLoader\"; \
             J48=\"weka.classifiers.trees.J48\"; \
             J48_OPTS=\"-v -t $TMP_TRAIN_DATA_ARFF -x 2 -c 1 -d %s\"; \
             $JAVA $JVM_OPTS $CSV_CONVERTER $TMP_TRAIN_DATA_CSV \
                   > $TMP_TRAIN_DATA_ARFF; \
             $EGREP '@attribute' $TMP_TRAIN_DATA_ARFF \
                   | $SED '2,$d' \
                   | $AWK '{for (i=3;i<=NF;i++) printf \"%%s \", $i; printf \"\\n\"}' \
                   > %s; \
             $JAVA $JVM_OPTS $J48 $J48_OPTS >> /dev/null; \
             rm -f $TMP_TRAIN_DATA_ARFF; \
             rm -f $TMP_TRAIN_DATA_CSV", 
             weka_jar_fname, HAPLO_JAVA, HAPLO_EGREP, HAPLO_AWK, HAPLO_SED,
             pid, tmp_data_csv_fname, model_fname, labels_fname);

    system(script);

    return NULL;
}


Error* train_weka_part_model
(
    const Vector_u32* labels, 
    const Matrix_i32* markers,
    const char*       labels_fname,
    const char*       model_fname,
    const char*       weka_jar_fname
)
{
    static char script[4096] = {0};
    static char tmp_data_csv_fname[256] = {0};
    const char* label_str;

    FILE*    fp;
    uint32_t pid;
    uint32_t s, num_samples;
    uint32_t m, num_markers;
    int      slen = 1024;
    char     str[slen];

    pid = getpid();

    snprintf(tmp_data_csv_fname, 255, "/tmp/.weka_part_train_data_csv-%u", pid);
    if (!(fp = fopen(tmp_data_csv_fname, "w")))
    {
        snprintf(str, slen, "%s: %s", tmp_data_csv_fname, 
                strerror(errno));
        return JWSC_EIO(str);
    }

    num_samples = markers->num_rows;
    num_markers = markers->num_cols;

    for (s = 0; s < num_samples; s++)
    {
        assert(lookup_haplo_group_label_from_index(&label_str, 
                    labels->elts[ s ]) == NULL);
        fprintf(fp, "%s", label_str);
        for (m = 0; m < num_markers; m++)
        {
            fprintf(fp, ",%u", markers->elts[ s ][ m ]);
        }
        fprintf(fp, "\n");
    }

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", tmp_data_csv_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    snprintf(script, 4096, 
            "WEKA_JAR=\"%s\"; \
             JAVA=\"%s\"; \
             EGREP=\"%s\"; \
             AWK=\"%s\"; \
             SED=\"%s\"; \
             TMP_TRAIN_DATA_ARFF=\"/tmp/.weka_part_train_data_arff-%u\"; \
             TMP_TRAIN_DATA_CSV=\"%s\"; \
             JVM_OPTS=\"-Xmx1024m -cp $WEKA_JAR\"; \
             CSV_CONVERTER=\"weka.core.converters.CSVLoader\"; \
             PART=\"weka.classifiers.rules.PART\"; \
             PART_OPTS=\"-v -t $TMP_TRAIN_DATA_ARFF -x 2 -c 1 -d %s\"; \
             $JAVA $JVM_OPTS $CSV_CONVERTER $TMP_TRAIN_DATA_CSV \
                   > $TMP_TRAIN_DATA_ARFF; \
             $EGREP '@attribute' $TMP_TRAIN_DATA_ARFF \
                   | $SED '2,$d' \
                   | $AWK '{for (i=3;i<=NF;i++) printf \"%%s \", $i; printf \"\\n\"}' \
                   > %s; \
             $JAVA $JVM_OPTS $PART $PART_OPTS >> /dev/null; \
             rm -f $TMP_TRAIN_DATA_ARFF; \
             rm -f $TMP_TRAIN_DATA_CSV", 
             weka_jar_fname, HAPLO_JAVA, HAPLO_EGREP, HAPLO_AWK, HAPLO_SED,
             pid, tmp_data_csv_fname, model_fname, labels_fname);

    system(script);

    return NULL;
}


Error* predict_labels_with_weka_j48_model
(
    Vector_u32**      labels_out,
    Vector_d**        confs_out,
    const Matrix_i32* markers,
    const char*       labels_fname,
    const char*       model_fname,
    const char*       weka_jar_fname
)
{
    static char script[4096] = {0};
    static char tmp_data_fname[256] = {0};
    static char tmp_result_fname[256] = {0};
    char label_buf[256] = {0};
    char* label_buf_ptr;
    char  c;

    FILE*    fp;
    uint32_t pid;
    uint32_t s, num_samples;
    uint32_t m, num_markers;
    uint32_t label;
    double   conf;
    char*    labels_str = NULL;
    Error*   err;
    int      slen = 1024;
    char     str[slen];

    if ((err = read_weka_model_labels(&labels_str, labels_fname)))
    {
        return err;
    }

    pid = getpid();
    num_samples = markers->num_rows;
    num_markers = markers->num_cols;

    create_vector_u32(labels_out, num_samples);
    create_vector_d(confs_out, num_samples);

    snprintf(tmp_data_fname, 256, "/tmp/.weka_j48_predict_data-%u", pid);
    snprintf(tmp_result_fname, 256, "/tmp/.weka_j48_predict_result-%u", pid);

    if ((fp = fopen(tmp_data_fname, "w")) == NULL)
    {
        snprintf(str, slen, "%s: %s", tmp_data_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    fprintf(fp, "@relation %s\n\n", tmp_data_fname);

    fprintf(fp, "@attribute label %s\n", labels_str);

    for (m = 0; m < num_markers; m++)
    {
        fprintf(fp, "@attribute %u numeric\n", m);
    }

    fprintf(fp, "\n@data\n");
    for (s = 0; s < num_samples; s++)
    {
        fprintf(fp, "?");
        for (m = 0; m < num_markers; m++)
        {
            if (markers->elts[ s ][ m ] > 0)
            {
                fprintf(fp, ",%u", markers->elts[ s ][ m ]);
            }
            else
            {
                fprintf(fp, ",?");
            }
        }
        fprintf(fp, "\n");
    }

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", tmp_data_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    snprintf(script, 4096, 
            "WEKA_JAR=\"%s\"; \
             JAVA=\"%s\"; \
             AWK=\"%s\"; \
             SED=\"%s\"; \
             TMP_DATA=\"%s\"; \
             TMP_RESULT=\"%s\"; \
             JVM_OPTS=\"-Xmx1024m -cp $WEKA_JAR\"; \
             J48=\"weka.classifiers.trees.J48\"; \
             J48_OPTS=\"-c 1 -T $TMP_DATA -l %s -p 0\"; \
             $JAVA $JVM_OPTS $J48 $J48_OPTS \
                    | $SED '$d' \
                    | $SED 's,^[^ ]\\+ \\(.*\\) \\([^ ]\\+\\) [^ ]\\+[ ]*$,\\1\\n\\2,' \
                    > $TMP_RESULT",
             weka_jar_fname, HAPLO_JAVA, HAPLO_AWK, HAPLO_SED, tmp_data_fname,
             tmp_result_fname, model_fname);
    system(script);

    if (!(fp = fopen(tmp_result_fname, "r")))
    {
        snprintf(str, slen, "%s: %s", tmp_result_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    for (s = 0; s < num_samples; s++)
    {
        label_buf_ptr = label_buf;
        while ((c = fgetc(fp)) != EOF && c != '\n' && 
                label_buf_ptr != (label_buf+256))
        {
            *label_buf_ptr = c;
            label_buf_ptr++;
        }
        if (label_buf_ptr != (label_buf+256))
        {
            *label_buf_ptr = 0;
        }

        if (fscanf(fp, "%lf\n", &conf) != 1)
        {
            snprintf(str, slen, "%s: Sample %d: %s", tmp_result_fname, 
                    s+1, "Invalid output from Weka J48");
            return JWSC_EIO(str);
        }
        if ((err = lookup_haplo_group_index_from_label(&label, label_buf)))
        {
            snprintf(str, slen, "%s: Sample %d: %s", tmp_result_fname, s+1,
                    err->msg);
            return JWSC_EIO(str);
        }

        (*labels_out)->elts[ s ] = label;
        (*confs_out)->elts[ s ] = conf;
    }

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", tmp_result_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    snprintf(script, 4096, "rm -f %s", tmp_data_fname);
    system(script);
    snprintf(script, 4096, "rm -f %s", tmp_result_fname);
    system(script);

    free(labels_str);

    return NULL;
}


Error* predict_labels_with_weka_part_model
(
    Vector_u32**      labels_out,
    Vector_d**        confs_out,
    const Matrix_i32* markers,
    const char*       labels_fname,
    const char*       model_fname,
    const char*       weka_jar_fname
)
{
    static char script[4096] = {0};
    static char tmp_data_fname[256] = {0};
    static char tmp_result_fname[256] = {0};
    char label_buf[256] = {0};
    char* label_buf_ptr;
    char  c;

    FILE*    fp;
    uint32_t pid;
    uint32_t s, num_samples;
    uint32_t m, num_markers;
    uint32_t label;
    double   conf;
    char*    labels_str = NULL;
    Error*   err;
    int      slen = 1024;
    char     str[slen];

    if ((err = read_weka_model_labels(&labels_str, labels_fname)))
    {
        return err;
    }

    pid = getpid();
    num_samples = markers->num_rows;
    num_markers = markers->num_cols;

    create_vector_u32(labels_out, num_samples);
    create_vector_d(confs_out, num_samples);

    snprintf(tmp_data_fname, 256, "/tmp/.weka_part_predict_data-%u", pid);
    snprintf(tmp_result_fname, 256, "/tmp/.weka_part_predict_result-%u", pid);

    if ((fp = fopen(tmp_data_fname, "w")) == NULL)
    {
        snprintf(str, slen, "%s: %s", tmp_data_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    fprintf(fp, "@relation %s\n\n", tmp_data_fname);

    fprintf(fp, "@attribute label %s\n", labels_str);

    for (m = 0; m < num_markers; m++)
    {
        fprintf(fp, "@attribute %u numeric\n", m);
    }

    fprintf(fp, "\n@data\n");
    for (s = 0; s < num_samples; s++)
    {
        fprintf(fp, "?");
        for (m = 0; m < num_markers; m++)
        {
            if (markers->elts[ s ][ m ] > 0)
            {
                fprintf(fp, ",%u", markers->elts[ s ][ m ]);
            }
            else
            {
                fprintf(fp, ",?");
            }
        }
        fprintf(fp, "\n");
    }

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", tmp_data_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    snprintf(script, 4096, 
            "WEKA_JAR=\"%s\"; \
             JAVA=\"%s\"; \
             AWK=\"%s\"; \
             SED=\"%s\"; \
             TMP_DATA=\"%s\"; \
             TMP_RESULT=\"%s\"; \
             JVM_OPTS=\"-Xmx1024m -cp $WEKA_JAR\"; \
             PART=\"weka.classifiers.rules.PART\"; \
             PART_OPTS=\"-c 1 -T $TMP_DATA -l %s -p 0\"; \
             $JAVA $JVM_OPTS $PART $PART_OPTS \
                    | $SED '$d' \
                    | $SED 's,^[^ ]\\+ \\(.*\\) \\([^ ]\\+\\) [^ ]\\+[ ]*$,\\1\\n\\2,' \
                    > $TMP_RESULT",
             weka_jar_fname, HAPLO_JAVA, HAPLO_AWK, HAPLO_SED, tmp_data_fname,
             tmp_result_fname, model_fname);
    system(script);

    if (!(fp = fopen(tmp_result_fname, "r")))
    {
        snprintf(str, slen, "%s: %s", tmp_result_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    for (s = 0; s < num_samples; s++)
    {
        label_buf_ptr = label_buf;
        while ((c = fgetc(fp)) != EOF && c != '\n' && 
                label_buf_ptr != (label_buf+256))
        {
            *label_buf_ptr = c;
            label_buf_ptr++;
        }
        if (label_buf_ptr != (label_buf+256))
        {
            *label_buf_ptr = 0;
        }

        if (fscanf(fp, "%lf\n", &conf) != 1)
        {
            snprintf(str, slen, "%s: Sample %d: %s", tmp_result_fname, 
                    s+1, "Invalid output from Weka PART");
            return JWSC_EIO(str);
        }
        if ((err = lookup_haplo_group_index_from_label(&label, label_buf)))
        {
            snprintf(str, slen, "%s: Sample %d: %s", tmp_result_fname, 
                    s+1, err->msg);
            return JWSC_EIO(str);
        }

        (*labels_out)->elts[ s ] = label;
        (*confs_out)->elts[ s ] = conf;
    }

    if (fclose(fp) != 0)
    {
        snprintf(str, slen, "%s: %s", tmp_result_fname, strerror(errno));
        return JWSC_EIO(str);
    }

    snprintf(script, 4096, "rm -f %s", tmp_data_fname);
    system(script);
    snprintf(script, 4096, "rm -f %s", tmp_result_fname);
    system(script);

    free(labels_str);

    return NULL;
}


static Error* read_weka_xml_doc
(
    xmlDoc**    xml_doc_out,
    const char* xml_fname,
    const char* dtd_fname
)
{
    xmlParserCtxt* xml_parse_ctxt;
    xmlValidCtxt*  xml_valid_ctxt;
    xmlDtd*        xml_dtd;
    int            slen = 256;
    char           str[slen];

    assert(xml_parse_ctxt = xmlNewParserCtxt());

    if (!(*xml_doc_out = xmlCtxtReadFile(xml_parse_ctxt, xml_fname, NULL, 0)))
    {
        snprintf(str, slen, "%s: %s", xml_fname, "Could not parse file");
        return JWSC_EARG(str);
    } 

    xmlFreeParserCtxt(xml_parse_ctxt);

    if (dtd_fname)
    {
        assert(xml_valid_ctxt = xmlNewValidCtxt());

        if (!(xml_dtd = xmlParseDTD(NULL, (xmlChar*)dtd_fname)))
        {
            snprintf(str, slen, "%s: %s", dtd_fname, "Could not parse DTD");
            return JWSC_EARG(str);
        }

        if (!xmlValidateDtd(xml_valid_ctxt, *xml_doc_out, xml_dtd))
        {
            snprintf(str, slen, "%s: %s", xml_fname, "XML file not valid");
            return JWSC_EARG(str);
        }

        xmlFreeValidCtxt(xml_valid_ctxt);
        xmlFreeDtd(xml_dtd);
    }

    return NULL;
}


static Error* create_weka_model_tree_from_xml_node
(
     Weka_model_tree**      tree_out, 
     const Weka_model_tree* parent,
     uint32_t               parent_label,
     xmlNode*               xml_node,
     const char*            model_dirname
)
{
    Weka_model_tree* tree;
    uint32_t i, j;
    uint32_t len;
    uint32_t label;
    uint32_t altlabel;
    uint32_t num_altlabels;
    xmlNode* it;
    xmlNode* itt;
    Error*   err;
    const char* fname;
    char*       fname_buf;

    if (*tree_out)
    {
        free_weka_model_tree(*tree_out);
    }

    assert(*tree_out = malloc(sizeof(Weka_model_tree)));
    tree = *tree_out;

    tree->parent       = parent;
    tree->parent_label = parent_label;
    tree->num_groups   = 0;
    tree->subtrees     = NULL;
    tree->labels       = NULL;
    tree->altlabels    = NULL;
    tree->labels_fname = NULL;
    tree->model_fname  = NULL;

    assert(XMLStrEqual(xml_node->name, "model"));

    for (it = xml_node->children; it; it = it->next)
    {
        if (it->type == XML_ELEMENT_NODE && XMLStrEqual(it->name, "group"))
        {
            tree->num_groups++;
        }
    }

    assert(tree->subtrees = calloc(tree->num_groups, sizeof(void*)));
    assert(tree->altlabels = calloc(tree->num_groups, sizeof(void*)));
    create_vector_u32(&(tree->labels), tree->num_groups);

    i = 0;
    for (it = xml_node->children; it; it = it->next)
    {
        j = 0;
        if (it->type == XML_ELEMENT_NODE && XMLStrEqual(it->name, "model-file"))
        {
            fname = (const char*) it->children->content;
            len = strlen(model_dirname) + strlen(fname) + 2;
            fname_buf = malloc(len * sizeof(char));
            snprintf(fname_buf, len, "%s/%s", model_dirname, fname);
            tree->model_fname = fname_buf;
        }
        else if (it->type == XML_ELEMENT_NODE && 
                XMLStrEqual(it->name, "labels-file"))
        {
            fname = (const char*) it->children->content;
            len = strlen(model_dirname) + strlen(fname) + 2;
            fname_buf = malloc(len * sizeof(char));
            snprintf(fname_buf, len, "%s/%s", model_dirname, fname);
            tree->labels_fname = fname_buf;
        }
        else if (it->type == XML_ELEMENT_NODE && XMLStrEqual(it->name, "group"))
        {
            num_altlabels = 0;
            for (itt = it->children; itt; itt = itt->next)
            {
                if (itt->type == XML_ELEMENT_NODE &&
                        XMLStrEqual(itt->name, "altlabel"))
                {
                    num_altlabels++;
                }
            }
            if (num_altlabels)
            {
                create_vector_u32(&(tree->altlabels[ i ]), num_altlabels);
            }

            for (itt = it->children; itt; itt = itt->next)
            {
                if (itt->type == XML_ELEMENT_NODE &&
                        XMLStrEqual(itt->name, "label"))
                {
                    if ((err = lookup_haplo_group_index_from_label(&label, 
                                    (const char*) itt->children->content)))
                    {
                        return err;
                    }
                    tree->labels->elts[ i ] = label;
                }
                else if (itt->type == XML_ELEMENT_NODE &&
                        XMLStrEqual(itt->name, "altlabel"))
                {
                    if ((err = lookup_haplo_group_index_from_label(&altlabel, 
                                    (const char*) itt->children->content)))
                    {
                        return err;
                    }
                    tree->altlabels[ i ]->elts[ j++ ] = altlabel;
                }
                else if (itt->type == XML_ELEMENT_NODE &&
                        XMLStrEqual(itt->name, "model"))
                {
                    if ((err = create_weka_model_tree_from_xml_node(
                                 &(tree->subtrees[ i ]), tree, label, itt,
                                 model_dirname)))
                    {
                        return err;
                    }
                }
            }
            i++;
        }
    }
    assert(i == tree->num_groups);

    return NULL;
}


static Error* create_model_training_data
(
    Vector_u32**      train_labels_out,
    Matrix_i32**      train_markers_out,
    const Vector_u32* data_labels,
    const Matrix_i32* data_markers,
    const Vector_u32* model_labels,
    Vector_u32*const* model_altlabels
)
{
    uint8_t     b;
    uint32_t    i, j, k;
    uint32_t    n;
    Vector_u32* train_labels = NULL;
    Matrix_i32* train_markers = NULL;

    copy_vector_u32(&train_labels, data_labels);
    copy_matrix_i32(&train_markers, data_markers);

    n = 0;
    for (i = 0; i < data_labels->num_elts; i++)
    {
        for (j = 0; j < model_labels->num_elts; j++)
        {
            if (is_ancestor(data_labels->elts[ i ], model_labels->elts[ j ]))
            {
                train_labels->elts[ n ] = model_labels->elts[ j ];
                copy_matrix_block_into_matrix_i32(train_markers, n, 0,
                        data_markers, i, 0, 1, data_markers->num_cols);
                n++;
                break;
            }
            else if (model_altlabels[ j ])
            {
                for (k = 0; k < model_altlabels[ j ]->num_elts; k++)
                {
                    if ((b = is_ancestor(data_labels->elts[ i ], 
                                model_altlabels[ j ]->elts[ k ])))
                    {
                        train_labels->elts[ n ] = model_labels->elts[ j ];
                        copy_matrix_block_into_matrix_i32(train_markers, n, 0,
                                data_markers, i, 0, 1, data_markers->num_cols);
                        n++;
                        break;
                    }
                }
                if (b)
                {
                    break;
                }
            }
        }
    }

    if (!n)
    {
        return JWSC_EARG("No data for model");
    }

    copy_vector_section_u32(train_labels_out, train_labels, 0, n);
    copy_matrix_block_i32(train_markers_out, train_markers, 0, 0, n, 
            train_markers->num_cols);

    free_vector_u32(train_labels);
    free_matrix_i32(train_markers);

    return NULL;
}


static Error* train_weka_j48_model_node
(
    Weka_model_node*  node,
    const Vector_u32* labels, 
    const Matrix_i32* markers,
    const char*       weka_jar_fname
)
{
    uint32_t i;
    Vector_u32* node_labels  = NULL;
    Matrix_i32* node_markers = NULL;
    Error*  err;
    int     slen = 256;
    char    str[slen];

    if ((err = create_model_training_data(&node_labels, &node_markers, labels,
                    markers, node->labels, node->altlabels)))
    {
        snprintf(str, slen, "%s: %s", node->model_fname, err->msg);
        return JWSC_EARG(str);
    }

    if ((err = train_weka_j48_model(node_labels, node_markers,
                    node->labels_fname, node->model_fname, weka_jar_fname)))
    {
        return err;
    }

    for (i = 0; i < node->num_groups; i++)
    {
        if (node->subtrees[ i ])
        {
            if ((err = train_weka_j48_model_node(node->subtrees[ i ], labels,
                        markers, weka_jar_fname)))
            {
                return err;
            }
        }
    }

    free_vector_u32(node_labels);
    free_matrix_i32(node_markers);

    return NULL;
}


static Error* train_weka_part_model_node
(
    Weka_model_node*  node,
    const Vector_u32* labels, 
    const Matrix_i32* markers,
    const char*       weka_jar_fname
)
{
    uint32_t i;
    Vector_u32* node_labels  = NULL;
    Matrix_i32* node_markers = NULL;
    Error*  err;
    int     slen = 256;
    char    str[slen];

    if ((err = create_model_training_data(&node_labels, &node_markers, labels,
                    markers, node->labels, node->altlabels)))
    {
        snprintf(str, slen, "%s: %s", node->model_fname, err->msg);
        return JWSC_EARG(str);
    }

    if ((err = train_weka_part_model(node_labels, node_markers,
                    node->labels_fname, node->model_fname, weka_jar_fname)))
    {
        return err;
    }

    for (i = 0; i < node->num_groups; i++)
    {
        if (node->subtrees[ i ])
        {
            if ((err = train_weka_part_model_node(node->subtrees[ i ], labels,
                        markers, weka_jar_fname)))
            {
                return err;
            }
        }
    }

    free_vector_u32(node_labels);
    free_matrix_i32(node_markers);

    return NULL;
}


Error* train_weka_j48_model_tree
(
    Weka_model_tree** tree_out,
    const Vector_u32* labels, 
    const Matrix_i32* markers,
    const char*       tree_xml_fname,
    const char*       tree_dtd_fname,
    const char*       model_dirname,
    const char*       weka_jar_fname
)
{
    xmlDoc*  xml_doc;
    xmlNode* xml_node;
    xmlNode* it;
    Error*   err;
    int      slen = 256;
    char     str[slen];

    assert(tree_xml_fname);

    if ((err = read_weka_xml_doc(&xml_doc, tree_xml_fname, tree_dtd_fname)))
    {
        return err;
    }

    for (it = xmlDocGetRootElement(xml_doc)->children; it; it = it->next)
    {
        if (it->type == XML_ELEMENT_NODE && XMLStrEqual(it->name, "model"))
        {
            xml_node = it;
            break;
        }
    }

    if ((err = create_weka_model_tree_from_xml_node(tree_out, NULL, 0, 
                    xml_node, model_dirname)))
    {
        snprintf(str, slen, "%s: %s", tree_xml_fname, err->msg);
        return JWSC_EARG(str);
    }

    if ((err = train_weka_j48_model_node(*tree_out, labels, markers,
                    weka_jar_fname)))
    {
        snprintf(str, slen, "%s: %s", tree_xml_fname, err->msg);
        return JWSC_EARG(str);
    }

    return NULL;
}


Error* train_weka_part_model_tree
(
    Weka_model_tree** tree_out,
    const Vector_u32* labels, 
    const Matrix_i32* markers,
    const char*       tree_xml_fname,
    const char*       tree_dtd_fname,
    const char*       model_dirname,
    const char*       weka_jar_fname
)
{
    xmlDoc*  xml_doc;
    xmlNode* xml_node;
    xmlNode* it;
    Error*   err;
    int      slen = 256;
    char     str[slen];

    assert(tree_xml_fname);

    if ((err = read_weka_xml_doc(&xml_doc, tree_xml_fname, tree_dtd_fname)))
    {
        return err;
    }

    for (it = xmlDocGetRootElement(xml_doc)->children; it; it = it->next)
    {
        if (it->type == XML_ELEMENT_NODE && XMLStrEqual(it->name, "model"))
        {
            xml_node = it;
            break;
        }
    }

    if ((err = create_weka_model_tree_from_xml_node(tree_out, NULL, 0, 
                    xml_node, model_dirname)))
    {
        snprintf(str, slen, "%s: %s", tree_xml_fname, err->msg);
        return JWSC_EARG(str);
    }

    if ((err = train_weka_part_model_node(*tree_out, labels, markers,
                    weka_jar_fname)))
    {
        snprintf(str, slen, "%s: %s", tree_xml_fname, err->msg);
        return JWSC_EARG(str);
    }

    return NULL;
}


static Error* recursively_predict_j48_labels_in_model_tree
(
    Vector_u32**           labels_out,
    Vector_d**             confs_in_out,
    const Weka_model_tree* tree,
    const Matrix_i32*      markers,
    const char*            weka_jar_fname
)
{
    uint32_t    s, num_samples;
    uint32_t    num_markers;
    uint32_t    n, N;
    uint32_t    i;
    uint32_t    subtree_label;
    Vector_d*   confs          = NULL;
    Vector_u32* sample_indices = NULL;
    Matrix_i32* markers_subset = NULL;
    Vector_u32* labels_subset  = NULL;
    Vector_d*   confs_subset   = NULL;
    Error*      err;

    num_samples = markers->num_rows;
    num_markers = markers->num_cols;

    assert((*confs_in_out)->num_elts == num_samples);

    if ((err = predict_labels_with_weka_j48_model(labels_out, &confs,
                    markers, tree->labels_fname, tree->model_fname,
                    weka_jar_fname)) != NULL)
    {
        return err;
    }
    for (s = 0; s < num_samples; s++)
    {
        (*confs_in_out)->elts[ s ] *= confs->elts[ s ];
    }

    create_vector_u32(&sample_indices, num_samples);

    for (i = 0; i < tree->num_groups; i++)
    {
        if (!tree->subtrees[ i ])
            continue;

        subtree_label = tree->subtrees[ i ]->parent_label;

        N = 0;
        for (s = 0; s < num_samples; s++)
        {
            if ((*labels_out)->elts[ s ] == subtree_label)
            {
                sample_indices->elts[ N++ ] = s;
            }
        }

        if (!N)
            continue;

        create_matrix_i32(&markers_subset, N, num_markers);
        create_vector_d(&confs_subset, N);

        for (n = 0; n < N; n++)
        {
            s = sample_indices->elts[ n ];
            copy_matrix_block_into_matrix_i32(markers_subset, n, 0,
                    markers, s, 0, 1, num_markers);
            confs_subset->elts[ n ] = (*confs_in_out)->elts[ s ];
        }

        if ((err = recursively_predict_j48_labels_in_model_tree(
                        &labels_subset, &confs_subset, tree->subtrees[ i ], 
                        markers_subset, weka_jar_fname)) != NULL)
        {
            return err;
        }

        for (n = 0; n < N; n++)
        {
            s = sample_indices->elts[ n ];
            (*labels_out)->elts[ s ] = labels_subset->elts[ n ];
            (*confs_in_out)->elts[ s ] = confs_subset->elts[ n ];
        }
    }

    free_vector_u32(sample_indices);
    free_matrix_i32(markers_subset);
    free_vector_u32(labels_subset);
    free_vector_d(confs_subset);

    return NULL;
}


static Error* recursively_predict_part_labels_in_model_tree
(
    Vector_u32**           labels_out,
    Vector_d**             confs_in_out,
    const Weka_model_tree* tree,
    const Matrix_i32*      markers,
    const char*            weka_jar_fname
)
{
    uint32_t    s, num_samples;
    uint32_t    num_markers;
    uint32_t    n, N;
    uint32_t    i;
    uint32_t    subtree_label;
    Vector_d*   confs          = NULL;
    Vector_u32* sample_indices = NULL;
    Matrix_i32* markers_subset = NULL;
    Vector_u32* labels_subset  = NULL;
    Vector_d*   confs_subset   = NULL;
    Error*      err;

    num_samples = markers->num_rows;
    num_markers = markers->num_cols;

    assert((*confs_in_out)->num_elts == num_samples);

    if ((err = predict_labels_with_weka_part_model(labels_out, &confs,
                    markers, tree->labels_fname, tree->model_fname,
                    weka_jar_fname)) != NULL)
    {
        return err;
    }
    for (s = 0; s < num_samples; s++)
    {
        (*confs_in_out)->elts[ s ] *= confs->elts[ s ];
    }

    create_vector_u32(&sample_indices, num_samples);

    for (i = 0; i < tree->num_groups; i++)
    {
        if (!tree->subtrees[ i ])
            continue;

        subtree_label = tree->subtrees[ i ]->parent_label;

        N = 0;
        for (s = 0; s < num_samples; s++)
        {
            if ((*labels_out)->elts[ s ] == subtree_label)
            {
                sample_indices->elts[ N++ ] = s;
            }
        }

        if (!N)
            continue;

        create_matrix_i32(&markers_subset, N, num_markers);
        create_vector_d(&confs_subset, N);

        for (n = 0; n < N; n++)
        {
            s = sample_indices->elts[ n ];
            copy_matrix_block_into_matrix_i32(markers_subset, n, 0,
                    markers, s, 0, 1, num_markers);
            confs_subset->elts[ n ] = (*confs_in_out)->elts[ s ];
        }

        if ((err = recursively_predict_part_labels_in_model_tree(
                        &labels_subset, &confs_subset, tree->subtrees[ i ], 
                        markers_subset, weka_jar_fname)) != NULL)
        {
            return err;
        }

        for (n = 0; n < N; n++)
        {
            s = sample_indices->elts[ n ];
            (*labels_out)->elts[ s ] = labels_subset->elts[ n ];
            (*confs_in_out)->elts[ s ] = confs_subset->elts[ n ];
        }
    }

    free_vector_u32(sample_indices);
    free_matrix_i32(markers_subset);
    free_vector_u32(labels_subset);
    free_vector_d(confs_subset);

    return NULL;
}


Error* predict_labels_with_weka_j48_model_tree
(
    Vector_u32**           labels_out,
    Vector_d**             confs_out,
    const Matrix_i32*      markers,
    const Weka_model_tree* tree,
    const char*            weka_jar_fname
)
{
    Error* err;

    create_init_vector_d(confs_out, markers->num_rows, 1.0);

    if ((err = recursively_predict_j48_labels_in_model_tree(labels_out,
                    confs_out, tree, markers, weka_jar_fname)) != NULL)
    {
        return err;
    }

    return NULL;
}


Error* predict_labels_with_weka_part_model_tree
(
    Vector_u32**           labels_out,
    Vector_d**             confs_out,
    const Matrix_i32*      markers,
    const Weka_model_tree* tree,
    const char*            weka_jar_fname
)
{
    Error* err;

    create_init_vector_d(confs_out, markers->num_rows, 1.0);

    if ((err = recursively_predict_part_labels_in_model_tree(labels_out,
                    confs_out, tree, markers, weka_jar_fname)) != NULL)
    {
        return err;
    }

    return NULL;
}


Error* read_weka_model_tree
(
    Weka_model_tree** tree_out,
    const char*       tree_xml_fname,
    const char*       tree_dtd_fname,
    const char*       model_dirname
)
{
    xmlDoc*  xml_doc;
    xmlNode* xml_node;
    xmlNode* it;
    Error*   err;
    int      slen = 256;
    char     str[slen];

    assert(tree_xml_fname && model_dirname);

    if ((err = read_weka_xml_doc(&xml_doc, tree_xml_fname, tree_dtd_fname)))
    {
        return err;
    }

    for (it = xmlDocGetRootElement(xml_doc)->children; it; it = it->next)
    {
        if (it->type == XML_ELEMENT_NODE && XMLStrEqual(it->name, "model"))
        {
            xml_node = it;
            break;
        }
    }

    if ((err = create_weka_model_tree_from_xml_node(tree_out, NULL, 0,
                    xml_node, model_dirname)))
    {
        snprintf(str, slen, "%s: %s", tree_xml_fname, err->msg);
        return JWSC_EARG(str);
    }

    return NULL;
}


void free_weka_model_tree(Weka_model_tree* tree)
{
    uint32_t i;

    if (!tree)
        return;

    for (i = 0; i < tree->num_groups; i++)
    {
        free_weka_model_tree(tree->subtrees[ i ]);
        free_vector_u32(tree->altlabels[ i ]);
    }

    free(tree->subtrees);
    free(tree->altlabels);
    free_vector_u32(tree->labels);
    free(tree->labels_fname);
    free(tree->model_fname);
    free(tree);
}