    // Copyright (C) 2010  Davis E. King (davis@dlib.net)
    // License: Boost Software License   See LICENSE.txt for the full license.
    #ifndef DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_H__
    #define DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_H__

    #include <vector>
    #include "../matrix.h"
    #include "cross_validate_multiclass_trainer_abstract.h"
    #include <sstream>

    namespace dlib
    {

    // ----------------------------------------------------------------------------------------

        template <
            typename dec_funct_type,
            typename sample_type,
            typename label_type
            >
        const matrix<double> test_multiclass_decision_function (
            const dec_funct_type& dec_funct,
            const std::vector<sample_type>& x_test,
            const std::vector<label_type>& y_test
        )
        {

            // make sure requires clause is not broken
            DLIB_ASSERT( is_learning_problem(x_test,y_test) == true,
                        "\tmatrix test_multiclass_decision_function()"
                        << "\n\t invalid inputs were given to this function"
                        << "\n\t is_learning_problem(x_test,y_test): "
                        << is_learning_problem(x_test,y_test));

            const std::vector<label_type> all_labels = dec_funct.get_labels();

            // make a lookup table that maps from labels to their index in all_labels
            std::map<label_type,unsigned long> label_to_int;
            for (unsigned long i = 0; i < all_labels.size(); ++i)
                label_to_int[all_labels[i]] = i;

            matrix<double, 0, 0, typename dec_funct_type::mem_manager_type> res;
            res.set_size(all_labels.size(), all_labels.size());

            res = 0;

            typename std::map<label_type,unsigned long>::const_iterator iter;

            // now test this trained object
            for (unsigned long i = 0; i < x_test.size(); ++i)
            {
                iter = label_to_int.find(y_test[i]);
                // ignore samples with labels that the decision function doesn't know about.
                if (iter == label_to_int.end())
                    continue;

                const unsigned long truth = iter->second;
                const unsigned long pred  = label_to_int[dec_funct(x_test[i])];

                res(truth,pred) += 1;
            }

            return res;
        }

    // ----------------------------------------------------------------------------------------

        class cross_validation_error : public dlib::error
        {
        public:
            cross_validation_error(const std::string& msg) : dlib::error(msg){};
        };

        template <
            typename trainer_type,
            typename sample_type,
            typename label_type
            >
        const matrix<double> cross_validate_multiclass_trainer (
            const trainer_type& trainer,
            const std::vector<sample_type>& x,
            const std::vector<label_type>& y,
            const long folds
        )
        {
            typedef typename trainer_type::mem_manager_type mem_manager_type;

            // make sure requires clause is not broken
            DLIB_ASSERT(is_learning_problem(x,y) == true &&
                        1 < folds && folds <= static_cast<long>(x.size()),
                "\tmatrix cross_validate_multiclass_trainer()"
                << "\n\t invalid inputs were given to this function"
                << "\n\t x.size(): " << x.size()
                << "\n\t folds:  " << folds
                << "\n\t is_learning_problem(x,y): " << is_learning_problem(x,y)
                );

            const std::vector<label_type> all_labels = select_all_distinct_labels(y);

            // count the number of times each label shows up
            std::map<label_type,long> label_counts;
            for (unsigned long i = 0; i < y.size(); ++i)
                label_counts[y[i]] += 1;

            // figure out how many samples from each class will be in the test and train splits
            std::map<label_type,long> num_in_test, num_in_train;
            for (typename std::map<label_type,long>::iterator i = label_counts.begin(); i != label_counts.end(); ++i)
            {
                const long in_test = i->second/folds;
                if (in_test == 0)
                {
                    std::ostringstream sout;
                    sout << "In dlib::cross_validate_multiclass_trainer(), the number of folds was larger" << std::endl;
                    sout << "than the number of elements of one of the training classes." << std::endl;
                    sout << "  folds: "<< folds << std::endl;
                    sout << "  size of class " << i->first << ": "<< i->second << std::endl;
                    throw cross_validation_error(sout.str());
                }
                num_in_test[i->first] = in_test;
                num_in_train[i->first] = i->second - in_test;
            }

            std::vector<sample_type> x_test, x_train;
            std::vector<label_type> y_test, y_train;

            matrix<double, 0, 0, mem_manager_type> res;

            std::map<label_type,long> next_test_idx;
            for (unsigned long i = 0; i < all_labels.size(); ++i)
                next_test_idx[all_labels[i]] = 0;

            label_type label;

            for (long i = 0; i < folds; ++i)
            {
                x_test.clear();
                y_test.clear();
                x_train.clear();
                y_train.clear();

                // load up the test samples
                for (unsigned long j = 0; j < all_labels.size(); ++j)
                {
                    label = all_labels[j];
                    long next = next_test_idx[label];

                    long cur = 0;
                    const long num_needed = num_in_test[label];
                    while (cur < num_needed)
                    {
                        if (y[next] == label)
                        {
                            x_test.push_back(x[next]);
                            y_test.push_back(label);
                            ++cur;
                        }
                        next = (next + 1)%x.size();
                    }

                    next_test_idx[label] = next;
                }

                // load up the training samples
                for (unsigned long j = 0; j < all_labels.size(); ++j)
                {
                    label = all_labels[j];
                    long next = next_test_idx[label];

                    long cur = 0;
                    const long num_needed = num_in_train[label];
                    while (cur < num_needed)
                    {
                        if (y[next] == label)
                        {
                            x_train.push_back(x[next]);
                            y_train.push_back(label);
                            ++cur;
                        }
                        next = (next + 1)%x.size();
                    }
                }

                try
                {
                    // do the training and testing
                    res += test_multiclass_decision_function(trainer.train(x_train,y_train),x_test,y_test);
                }
                catch (invalid_nu_error&)
                {
                    // just ignore cases which result in an invalid nu
                }

            } // for (long i = 0; i < folds; ++i)

            return res;
        }

    }

    // ----------------------------------------------------------------------------------------

    #endif // DLIB_CROSS_VALIDATE_MULTICLASS_TRaINER_H__