source: src/main/java/weka/estimators/CheckEstimator.java @ 19

/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 *    CheckEstimator.java
 *    Copyright (C) 1999 University of Waikato, Hamilton, New Zealand
 *
 */

package weka.estimators;

import weka.core.Attribute;
import weka.core.FastVector;
import weka.core.Instance;
import weka.core.Instances;
import weka.core.Option;
import weka.core.OptionHandler;
import weka.core.RevisionHandler;
import weka.core.RevisionUtils;
import weka.core.TestInstances;
import weka.core.Utils;
import weka.core.WeightedInstancesHandler;

import java.util.Enumeration;
import java.util.Random;
import java.util.Vector;

/**
 * Class for examining the capabilities and finding problems with 
 * estimators. If you implement a estimator using the WEKA.libraries,
 * you should run the checks on it to ensure robustness and correct
 * operation. Passing all the tests of this object does not mean
 * bugs in the estimator don't exist, but this will help find some
 * common ones. <p/>
 * 
 * Typical usage: <p/>
 * <code>java weka.estimators.CheckEstimator -W estimator_name 
 * estimator_options </code><p/>
 * 
 * This class uses code from the CheckEstimatorClass
 * ATTENTION! Current estimators can only 
 * 1. split on a nominal class attribute
 * 2. build estimators for nominal and numeric attributes
 * 3. build estimators independendly of the class type
 * The functionality to test on other class and attribute types
 * is left in big parts in the code. 
 * 
 * CheckEstimator reports on the following:
 * <ul>
 *    <li> Estimator abilities 
 *      <ul>
 *         <li> Possible command line options to the estimator </li>
 *         <li> Whether the estimator can predict nominal, numeric, string, 
 *              date or relational class attributes. Warnings will be displayed if 
 *              performance is worse than ZeroR </li>
 *         <li> Whether the estimator can be trained incrementally </li>
 *         <li> Whether the estimator can build estimates for numeric attributes </li>
 *         <li> Whether the estimator can handle nominal attributes </li>
 *         <li> Whether the estimator can handle string attributes </li>
 *         <li> Whether the estimator can handle date attributes </li>
 *         <li> Whether the estimator can handle relational  attributes </li>
 *         <li> Whether the estimator build estimates for multi-instance data </li>
 *         <li> Whether the estimator can handle missing attribute values </li>
 *         <li> Whether the estimator can handle missing class values </li>
 *         <li> Whether a nominal estimator only handles 2 class problems </li>
 *         <li> Whether the estimator can handle instance weights </li>
 *      </ul>
 *    </li>
 *    <li> Correct functioning 
 *      <ul>
 *         <li> Correct initialisation during addvalues (i.e. no result
 *              changes when addValues called repeatedly) </li>
 *         <li> Whether incremental training produces the same results
 *              as during non-incremental training (which may or may not 
 *              be OK) </li>
 *         <li> Whether the estimator alters the data pased to it 
 *              (number of instances, instance order, instance weights, etc) </li>
 *      </ul>
 *    </li>
 *    <li> Degenerate cases 
 *      <ul>
 *         <li> building estimator with zero training instances </li>
 *         <li> all but one attribute attribute values missing </li>
 *         <li> all attribute attribute values missing </li>
 *         <li> all but one class values missing </li>
 *         <li> all class values missing </li>
 *      </ul>
 *    </li>
 * </ul>
 * Running CheckEstimator with the debug option set will output the 
 * training and test datasets for any failed tests.<p/>
 *
 * The <code>weka.estimators.AbstractEstimatorTest</code> uses this
 * class to test all the estimators. Any changes here, have to be 
 * checked in that abstract test class, too. <p/>
 *
 <!-- options-start -->
 * Valid options are: <p/>
 * 
 * <pre> -D
 *  Turn on debugging output.</pre>
 * 
 * <pre> -S
 *  Silent mode - prints nothing to stdout.</pre>
 * 
 * <pre> -N &lt;num&gt;
 *  The number of instances in the datasets (default 100).</pre>
 * 
 * <pre> -W
 *  Full name of the estimator analysed.
 *  eg: weka.estimators.NormalEstimator</pre>
 * 
 * <pre> 
 * Options specific to estimator weka.estimators.NormalEstimator:
 * </pre>
 * 
 * <pre> -D
 *  If set, estimator is run in debug mode and
 *  may output additional info to the console</pre>
 * 
 <!-- options-end -->
 *
 * Options after -- are passed to the designated estimator.<p/>
 *
 * @author Len Trigg (trigg@cs.waikato.ac.nz)
 * @author FracPete (fracpete at waikato dot ac dot nz)
 * @version $Revision: 4997 $
 * @see TestInstances
 */
public class CheckEstimator implements OptionHandler, RevisionHandler {

  /*
   * Note about test methods:
   * - methods return array of booleans
   * - first index: success or not
   * - second index: acceptable or not (e.g., Exception is OK)
   * - in case the performance is worse than that of ZeroR both indices are true
   *
   * FracPete (fracpete at waikato dot ac dot nz)
   */
  
  /** a class for postprocessing the test-data 
   */
  public class PostProcessor
    implements RevisionHandler {
    /**
     * Provides a hook for derived classes to further modify the data. Currently,
     * the data is just passed through.
     * 
     * @param data      the data to process
     * @return          the processed data
     */
    protected Instances process(Instances data) {
      return data;
    }
    
    /**
     * Returns the revision string.
     * 
     * @return          the revision
     */
    public String getRevision() {
      return RevisionUtils.extract("$Revision: 4997 $");
    }
  }
  
  /*** The estimator to be examined */
  protected Estimator m_Estimator = (Estimator) new weka.estimators.NormalEstimator(0.000001);
  
  /** The options to be passed to the base estimator. */
  protected String[] m_EstimatorOptions;
  
  /** The results of the analysis as a string */
  protected String m_AnalysisResults;
  
  /** Debugging mode, gives extra output if true */
  protected boolean m_Debug = false;
  
  /** Silent mode, for no output at all to stdout */
  protected boolean m_Silent = false;
  
  /** The number of instances in the datasets */
  protected int m_NumInstances = 100;
  
  /** for post-processing the data even further */
  protected PostProcessor m_PostProcessor = null;
  
  /** whether classpath problems occurred */
  protected boolean m_ClasspathProblems = false;
  
  /**
   * class that contains info about the attribute types the estimator can estimate
   * estimator work on one attribute only
   */
  public static class AttrTypes
    implements RevisionHandler {
    
    boolean nominal = false;
    boolean numeric = false; 
    boolean string = false;
    boolean date = false;
    boolean relational = false;
        
    AttrTypes() {
    }

    AttrTypes (AttrTypes newTypes) {
      nominal = newTypes.nominal;
      numeric = newTypes.numeric;
      string = newTypes.string;
      date = newTypes.date;
      relational = newTypes.relational;
    }
                        
    AttrTypes (int type) {
      if (type == Attribute.NOMINAL) nominal = true;
      if (type == Attribute.NUMERIC) numeric = true;
      if (type == Attribute.STRING) string = true;
      if (type == Attribute.DATE) date = true;
      if (type == Attribute.RELATIONAL) relational = true;
    }

    int getSetType() throws Exception {                 
      int sum = 0;
      int type = -1;
      if (nominal) { sum ++; type = Attribute.NOMINAL; }
      if (numeric) { sum ++; type = Attribute.NUMERIC; }
      if (string) { sum ++; type = Attribute.STRING; }
      if (date) { sum ++; type = Attribute.DATE; }
      if (relational) { sum ++; type = Attribute.RELATIONAL; }
      if (sum > 1)
        throw new Exception("Expected to have only one type set used wrongly.");
      if (type < 0)
        throw new Exception("No type set.");
      return type;
    }

    boolean oneIsSet() {
      return (nominal || numeric || string || date || relational);
    }

    public Vector getVectorOfAttrTypes() {
      Vector attrs = new Vector();
      if (nominal) attrs.add(new Integer(Attribute.NOMINAL));
      if (numeric) attrs.add(new Integer(Attribute.NUMERIC));
      if (string) attrs.add(new Integer(Attribute.STRING));
      if (date) attrs.add(new Integer(Attribute.DATE));
      if (relational) attrs.add(new Integer(Attribute.RELATIONAL));
      return attrs;
    }   
    
    /**
     * Returns the revision string.
     * 
     * @return          the revision
     */
    public String getRevision() {
      return RevisionUtils.extract("$Revision: 4997 $");
    }
  }

  /**
   * public class that contains info about the chosen attribute type
   * estimator work on one attribute only
   */
  public static class EstTypes
    implements RevisionHandler {
    
    boolean incremental = false;
    boolean weighted = false;
    boolean supervised = false;

    /**
     * Constructor
     */
    public EstTypes () {
    }

    /**
     * Constructor
     */
    public EstTypes (boolean i, boolean w, boolean s) {
      incremental = i;
      weighted    = w;
      supervised  = s;
    }
    
    /**
     * Returns the revision string.
     * 
     * @return          the revision
     */
    public String getRevision() {
      return RevisionUtils.extract("$Revision: 4997 $");
    }
  }

  /**
   * Returns an enumeration describing the available options.
   *
   * @return an enumeration of all the available options.
   */
  public Enumeration listOptions() {
    
    Vector newVector = new Vector(2);
    
    newVector.addElement(new Option(
        "\tTurn on debugging output.",
        "D", 0, "-D"));
    
    newVector.addElement(new Option(
        "\tSilent mode - prints nothing to stdout.",
        "S", 0, "-S"));
    
    newVector.addElement(new Option(
        "\tThe number of instances in the datasets (default 100).",
        "N", 1, "-N <num>"));
    
    newVector.addElement(new Option(
        "\tFull name of the estimator analysed.\n"
        +"\teg: weka.estimators.NormalEstimator",
        "W", 1, "-W"));
    
    if ((m_Estimator != null) 
        && (m_Estimator instanceof OptionHandler)) {
      newVector.addElement(new Option("", "", 0, 
          "\nOptions specific to estimator "
          + m_Estimator.getClass().getName()
          + ":"));
      Enumeration enu = ((OptionHandler)m_Estimator).listOptions();
      while (enu.hasMoreElements())
        newVector.addElement(enu.nextElement());
    }
    
    return newVector.elements();
  }
  
  /**
   * Parses a given list of options. 
   *
   <!-- options-start -->
   * Valid options are: <p/>
   * 
   * <pre> -D
   *  Turn on debugging output.</pre>
   * 
   * <pre> -S
   *  Silent mode - prints nothing to stdout.</pre>
   * 
   * <pre> -N &lt;num&gt;
   *  The number of instances in the datasets (default 100).</pre>
   * 
   * <pre> -W
   *  Full name of the estimator analysed.
   *  eg: weka.estimators.NormalEstimator</pre>
   * 
   * <pre> 
   * Options specific to estimator weka.estimators.NormalEstimator:
   * </pre>
   * 
   * <pre> -D
   *  If set, estimator is run in debug mode and
   *  may output additional info to the console</pre>
   * 
   <!-- options-end -->
   *
   * @param options the list of options as an array of strings
   * @throws Exception if an option is not supported
   */
  public void setOptions(String[] options) throws Exception {
    String      tmpStr;
    
    setDebug(Utils.getFlag('D', options));
    
    setSilent(Utils.getFlag('S', options));
    
    tmpStr = Utils.getOption('N', options);
    if (tmpStr.length() != 0)
      setNumInstances(Integer.parseInt(tmpStr));
    else
      setNumInstances(100);
    
    tmpStr = Utils.getOption('W', options);
    if (tmpStr.length() == 0)
      throw new Exception("A estimator must be specified with the -W option.");
    setEstimator(Estimator.forName(tmpStr, Utils.partitionOptions(options)));
  }
  
  /**
   * Gets the current settings of the CheckEstimator.
   *
   * @return an array of strings suitable for passing to setOptions
   */
  public String[] getOptions() {
    Vector        result;
    String[]      options;
    int           i;
    
    result = new Vector();
    
    if (getDebug())
      result.add("-D");
    
    if (getSilent())
      result.add("-S");
    
    result.add("-N");
    result.add("" + getNumInstances());
    
    if (getEstimator() != null) {
      result.add("-W");
      result.add(getEstimator().getClass().getName());
    }
    
    if ((m_Estimator != null) && (m_Estimator instanceof OptionHandler))
      options = ((OptionHandler) m_Estimator).getOptions();
    else
      options = new String[0];
    
    if (options.length > 0) {
      result.add("--");
      for (i = 0; i < options.length; i++)
        result.add(options[i]);
    }
    
    return (String[]) result.toArray(new String[result.size()]);
  }
  
  /**
   * sets the PostProcessor to use
   * 
   * @param value       the new PostProcessor
   * @see #m_PostProcessor
   */
  public void setPostProcessor(PostProcessor value) {
    m_PostProcessor = value;
  }
  
  /**
   * returns the current PostProcessor, can be null
   * 
   * @return            the current PostProcessor
   */
  public PostProcessor getPostProcessor() {
    return m_PostProcessor;
  }
  
  /**
   * returns TRUE if the estimator returned a "not in classpath" Exception
   * 
   * @return    true if CLASSPATH problems occurred
   */
  public boolean hasClasspathProblems() {
    return m_ClasspathProblems;
  }
  
  /**
   * Begin the tests, reporting results to System.out
   */
  public void doTests() {
    
    if (getEstimator() == null) {
      println("\n=== No estimator set ===");
      return;
    }
    println("\n=== Check on Estimator: "
        + getEstimator().getClass().getName()
        + " ===\n");
    
    m_ClasspathProblems = false;

    // Start tests with test for options
    canTakeOptions();

    // test what type of estimator it is 
    EstTypes estTypes = new EstTypes();
    estTypes.incremental = incrementalEstimator()[0];
    estTypes.weighted = weightedInstancesHandler()[0];
    estTypes.supervised = supervisedEstimator()[0];
   
    // in none of the estimators yet the functionality is depending on the class type
    // since this could change the basic structure taken from checkclassifiers is kept here
    int classType = Attribute.NOMINAL;
    AttrTypes attrTypes = testsPerClassType(classType, estTypes);
    
 
    // only nominal class can be split up so far
    canSplitUpClass(attrTypes, classType);
 }
  
  
  /**
   * Set debugging mode
   *
   * @param debug true if debug output should be printed
   */
  public void setDebug(boolean debug) {
    m_Debug = debug;

    // disable silent mode, if necessary
    if (getDebug())
      setSilent(false);
  }
  
  /**
   * Get whether debugging is turned on
   *
   * @return true if debugging output is on
   */
  public boolean getDebug() {
    return m_Debug;
  }
  
  /**
   * Set slient mode, i.e., no output at all to stdout
   *
   * @param value whether silent mode is active or not
   */
  public void setSilent(boolean value) {
    m_Silent = value;
  }
  
  /**
   * Get whether silent mode is turned on
   *
   * @return true if silent mode is on
   */
  public boolean getSilent() {
    return m_Silent;
  }
  
  /**
   * Sets the number of instances to use in the datasets (some estimators
   * might require more instances).
   *
   * @param value the number of instances to use
   */
  public void setNumInstances(int value) {
    m_NumInstances = value;
  }
  
  /**
   * Gets the current number of instances to use for the datasets.
   *
   * @return the number of instances
   */
  public int getNumInstances() {
    return m_NumInstances;
  }
  
  /**
   * Set the estimator for boosting. 
   *
   * @param newEstimator the Estimator to use.
   */
  public void setEstimator(Estimator newEstimator) {
    m_Estimator = newEstimator;
  }
  
  /**
   * Get the estimator used as the estimator
   *
   * @return the estimator used as the estimator
   */
  public Estimator getEstimator() {
    return m_Estimator;
  }
  
  /**
   * prints the given message to stdout, if not silent mode
   * 
   * @param msg         the text to print to stdout
   */
  protected void print(Object msg) {
    if (!getSilent())
      System.out.print(msg);
  }
  
  /**
   * prints the given message (+ LF) to stdout, if not silent mode
   * 
   * @param msg         the message to println to stdout
   */
  protected void println(Object msg) {
    print(msg + "\n");
  }
  
  /**
   * prints a LF to stdout, if not silent mode
   */
  protected void println() {
    print("\n");
  }
  
  /**
   * Run a battery of tests for a given class attribute type
   *
   * @param classType true if the class attribute should be numeric
   * @param estTypes types the estimator is, like incremental, weighted, supervised etc
   * @return attribute types estimator can work with
   */
  protected AttrTypes testsPerClassType(int classType, EstTypes estTypes) {
    
    // in none of the estimators yet is the estimation depending on the class type
    // since this could change the basic structure taken from checkclassifiers is kept here
    
    // test A: simple test - if can estimate
    AttrTypes attrTypes = new AttrTypes();
    AttrTypes at = new AttrTypes(Attribute.NOMINAL);
    attrTypes.nominal = canEstimate(at, estTypes.supervised, classType)[0];
    at = new AttrTypes(Attribute.NUMERIC);
    attrTypes.numeric = canEstimate(at, estTypes.supervised, classType)[0];
    attrTypes.string = false;
    attrTypes.date = false;
    attrTypes.relational = false;
    
//  if (!multiInstance)
//  PRel = canEstimate(false, false, false, false,  true, classType)[0];
//  else
//  PRel = false;
    
//  one of the attribute types succeeded
    
    if (attrTypes.oneIsSet()) {
      Vector attributesSet = attrTypes.getVectorOfAttrTypes();
      
      // make tests for each attribute
      for (int i = 0; i < attributesSet.size(); i++) {
        AttrTypes workAttrTypes = new AttrTypes(((Integer) attributesSet.elementAt(i)).intValue());
        
        // test B: weights change estimate or not
        if (estTypes.weighted)
          instanceWeights(workAttrTypes, classType);
        
        if (classType == Attribute.NOMINAL) {
          int numClasses = 4;
          canHandleNClasses(workAttrTypes, numClasses);
        }
        
        // tests with class not the last attribute and the attribute not the first
        
        //   if (!multiInstance) {
        int numAtt = 4; 
        
        canHandleClassAsNthAttribute(workAttrTypes, numAtt, 0, classType, 1);
        
        //TODOTODOcanHandleAttrAsNthAttribute(workAttrTypes, numAtt, 2, classType);
        //}
        
        canHandleZeroTraining(workAttrTypes, classType);
        boolean handleMissingAttributes = canHandleMissing(workAttrTypes, 
            classType, true, false, 20)[0];
        if (handleMissingAttributes)
          canHandleMissing(workAttrTypes, classType, true, false, 100);
        
        boolean handleMissingClass = canHandleMissing(workAttrTypes, 
            classType, 
            false, true, 20)[0];
        if (handleMissingClass)
          canHandleMissing(workAttrTypes, classType, false, true, 100);
        
        correctBuildInitialisation(workAttrTypes, classType);
        datasetIntegrity(workAttrTypes, classType,
            handleMissingAttributes, handleMissingClass);
        
        if (estTypes.incremental)
          incrementingEquality(workAttrTypes, classType);
      }
    }
    return attrTypes;
  }
  
  /**
   * Checks whether the scheme can take command line options.
   *
   * @return index 0 is true if the estimator can take options
   */
  protected boolean[] canTakeOptions() {
    
    boolean[] result = new boolean[2];
    
    print("options...");
    if (m_Estimator instanceof OptionHandler) {
      println("yes");
      if (m_Debug) {
        println("\n=== Full report ===");
        Enumeration enu = ((OptionHandler)m_Estimator).listOptions();
        while (enu.hasMoreElements()) {
          Option option = (Option) enu.nextElement();
          print(option.synopsis() + "\n" 
              + option.description() + "\n");
        }
        println("\n");
      }
      result[0] = true;
    }
    else {
      println("no");
      result[0] = false;
    }
    
    return result;
  }
  
  /**
   * Checks whether the scheme can build models incrementally.
   *
   * @return index 0 is true if the estimator can train incrementally
   */
  protected boolean[] incrementalEstimator() {
    
    boolean[] result = new boolean[2];
    
    print("incremental estimator...");
    if (m_Estimator instanceof IncrementalEstimator) {
      println("yes");
      result[0] = true;
    }
    else {
      println("no");
      result[0] = false;
    }
    
    return result;
  }
  
  /**
   * Checks whether the scheme says it can handle instance weights.
   *
   * @return true if the estimator handles instance weights
   */
  protected boolean[] weightedInstancesHandler() {
    
    boolean[] result = new boolean[2];
    
    print("weighted instances estimator...");
    if (m_Estimator instanceof WeightedInstancesHandler) {
      println("yes");
      result[0] = true;
    }
    else {
      println("no");
      result[0] = false;
    }
    
    return result;
  }

  /**
   * Checks whether the estimator is supervised.
   *
   * @return true if the estimator handles instance weights
   */
  protected boolean[] supervisedEstimator() {
    boolean[] result = new boolean[2];
    result[0] = false;
    return result;
  }

  /**
   * Checks basic estimation of one attribute of the scheme, for simple non-troublesome
   * datasets.
   *
   * @param attrTypes the types the estimator can work with
   * @param classType the class type (NOMINAL, NUMERIC, etc.)
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] canEstimate(AttrTypes attrTypes, boolean supervised, int classType) {
    
  // supervised is ignored, no supervised estimators used yet
    
    print("basic estimation");
    printAttributeSummary(attrTypes, classType);
    print("...");
    FastVector accepts = new FastVector();
    accepts.addElement("nominal");
    accepts.addElement("numeric");
    accepts.addElement("string");
    accepts.addElement("date");
    accepts.addElement("relational");
    accepts.addElement("not in classpath");
    int numTrain = getNumInstances(), numTest = getNumInstances(), 
    numClasses = 2, missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    int numAtts = 1, attrIndex = 0;

    return runBasicTest(attrTypes, numAtts, attrIndex,
                        classType, 
                        missingLevel, attributeMissing, classMissing,
                        numTrain, numTest, numClasses, 
                        accepts);
  }
  
  /**
   * Checks basic estimation of one attribute of the scheme, for simple non-troublesome
   * datasets.
   *
   * @param attrTypes the types the estimator can work with
   * @param classType the class type (NOMINAL, NUMERIC, etc.)
    */
  protected void canSplitUpClass(AttrTypes attrTypes, int classType) {
    
    if (attrTypes.nominal)
      canSplitUpClass(Attribute.NOMINAL, classType);
    if (attrTypes.numeric)
      canSplitUpClass(Attribute.NUMERIC, classType);
  }
  
  /**
   * Checks basic estimation of one attribute of the scheme, for simple non-troublesome
   * datasets.
   *
   * @param attrType the type of the estimator
   * @param classType the class type (NOMINAL, NUMERIC, etc.)
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] canSplitUpClass(int attrType, int classType) {
    
    boolean[] result = new boolean[2];

    FastVector accepts = new FastVector();
    accepts.addElement("not in classpath");

    // supervised is ignored, no supervised estimators used yet
    print("split per class type ");
    printAttributeSummary(attrType, Attribute.NOMINAL);
    print("...");
      
    int numTrain = getNumInstances(), numTest = getNumInstances(), 
    numClasses = 2;
    boolean attributeMissing = false, classMissing = false;
    int numAtts = 3, attrIndex = 0, classIndex = 1;
    Instances train = null;
    Vector test;
    Estimator estimator = null;
    boolean built = false;
    
    try {
      AttrTypes at = new AttrTypes(attrType);
      train = makeTestDataset(42, numTrain, numAtts, at,
          numClasses, classType, classIndex);
      
       // prepare training data set and test value list
      test = makeTestValueList(24, numTest, train, attrIndex,
          attrType);
      
       estimator = Estimator.makeCopies(getEstimator(), 1)[0];
    } catch (Exception ex) {
      ex.printStackTrace();
      throw new Error("Error setting up for tests: " + ex.getMessage());
    }
    try {
      estimator.addValues(train, attrIndex, classType, classIndex);
      built = true;
      
      testWithTestValues(estimator, test);
      
      println("yes");
      result[0] = true;
    } 
    catch (Exception ex) {
      boolean acceptable = false;
      String msg;
      if (ex.getMessage() == null)
        msg = "";
      else
        msg = ex.getMessage().toLowerCase();
      if (msg.indexOf("not in classpath") > -1)
        m_ClasspathProblems = true;
      
      for (int i = 0; i < accepts.size(); i++) {
        if (msg.indexOf((String)accepts.elementAt(i)) >= 0) {
          acceptable = true;
        }
      }
      
      println("no" + (acceptable ? " (OK error message)" : ""));
      result[1] = acceptable;
      
      
      if (m_Debug) {
        println("\n=== Full Report ===");
        print("Problem during");
        if (built) {
          print(" testing");
        } else {
          print(" training");
        }
        println(": " + ex.getMessage() + "\n");
        if (!acceptable) {
          if (accepts.size() > 0) {
            print("Error message doesn't mention ");
            for (int i = 0; i < accepts.size(); i++) {
              if (i != 0) {
                print(" or ");
              }
              print('"' + (String)accepts.elementAt(i) + '"');
            }
          }
          println("here are the datasets:\n");
          println("=== Train Dataset ===\n"
              + train.toString() + "\n");
          println("=== Test Dataset ===\n"
              + test.toString() + "\n\n");
        }
        
      }
    }
    return result;
   }
  
  /**
   * Checks whether nominal schemes can handle more than two classes.
   * If a scheme is only designed for two-class problems it should
   * throw an appropriate exception for multi-class problems.
   *
   * @param attrTypes attribute types the estimator excepts 
   * @param numClasses the number of classes to test
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] canHandleNClasses(AttrTypes attrTypes, int numClasses) {
    
    print("more than two class problems");
    printAttributeSummary(attrTypes, Attribute.NOMINAL);
    print("...");

    FastVector accepts = new FastVector();
    accepts.addElement("number");
    accepts.addElement("class");

    int numTrain = getNumInstances(), numTest = getNumInstances(), 
      missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    int numAttr = 1, attrIndex = 0;

    return runBasicTest(attrTypes,
                        numAttr, attrIndex,
                        Attribute.NOMINAL,
                        missingLevel, attributeMissing, classMissing,
                        numTrain, numTest, numClasses, 
                        accepts);
  }
  
  /**
   * Checks whether the scheme can handle class attributes as Nth attribute.
   *
   * @param attrTypes the attribute types the estimator accepts
   * @param numAtts of attributes
   * @param attrIndex the index of the attribute
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @param classIndex the index of the class attribute (0-based, -1 means last attribute)
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   * @see TestInstances#CLASS_IS_LAST
   */
  protected boolean[] canHandleClassAsNthAttribute(AttrTypes attrTypes,
                                                   int numAtts,
                                                   int attrIndex,
                                                   int classType,
                                                   int classIndex) {
    
    if (classIndex == TestInstances.CLASS_IS_LAST)
      print("class attribute as last attribute");
    else
      print("class attribute as " + (classIndex + 1) + ". attribute");
    printAttributeSummary(attrTypes, classType);
    print("...");
    FastVector accepts = new FastVector();
    int numTrain = getNumInstances(), numTest = getNumInstances(), numClasses = 2, 
    missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    
    return runBasicTest(attrTypes,
                        numAtts, attrIndex,
                        classType, classIndex,
                        missingLevel, attributeMissing, classMissing,
                        numTrain, numTest, numClasses, 
                        accepts);
  }
  
  /**
   * Checks whether the scheme can handle zero training instances.
   *
   * @param attrTypes attribute types that can be estimated
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] canHandleZeroTraining(AttrTypes attrTypes, int classType) {
    
    print("handle zero training instances");
    printAttributeSummary(attrTypes, classType);

    print("...");
    FastVector accepts = new FastVector();
    accepts.addElement("train");
    accepts.addElement("value");
    int numTrain = 0, numTest = getNumInstances(), numClasses = 2, 
    missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    int numAtts = 1;
    int attrIndex = 0;
    return runBasicTest(
              attrTypes, numAtts, attrIndex,
              classType, 
              missingLevel, attributeMissing, classMissing,
              numTrain, numTest, numClasses, 
              accepts);
  }
  
  /**
   * Checks whether the scheme correctly initialises models when 
   * buildEstimator is called. This test calls buildEstimator with
   * one training dataset and records performance on a test set. 
   * buildEstimator is then called on a training set with different
   * structure, and then again with the original training set. The
   * performance on the test set is compared with the original results
   * and any performance difference noted as incorrect build initialisation.
   *
   * @param attrTypes attribute types that can be estimated
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @return index 0 is true if the test was passed, index 1 is true if the
   *         scheme performs worse than ZeroR, but without error (index 0 is
   *         false)
   */
  protected boolean[] correctBuildInitialisation(AttrTypes attrTypes,
                                                 int classType) {

    boolean[] result = new boolean[2];
    
    print("correct initialisation during buildEstimator");
    printAttributeSummary(attrTypes, classType);

    print("...");
    int numTrain = getNumInstances(), numTest = getNumInstances(), 
    numClasses = 2, missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    
    Instances train1 = null;
    Instances test1 = null;
    Instances train2 = null;
    Instances test2 = null;
    Estimator estimator = null;
    Estimator estimator1 = null;
    
    boolean built = false;
    int stage = 0;
    int attrIndex1 = 1;
    int attrIndex2 = 2;

    try {
      
      // Make two sets of train/test splits with different 
      // numbers of attributes
      train1 = makeTestDataset(42, numTrain, 2, attrTypes,
                               numClasses, 
                               classType);
      train2 = makeTestDataset(84, numTrain, 3, attrTypes,
                               numClasses, 
                               classType);
      if (missingLevel > 0) {
        addMissing(train1, missingLevel, attributeMissing, classMissing, attrIndex1);
        addMissing(train2, missingLevel, attributeMissing, classMissing, attrIndex2);
      }
      
      estimator = Estimator.makeCopies(getEstimator(), 1)[0];
    } catch (Exception ex) {
      throw new Error("Error setting up for tests: " + ex.getMessage());
    }
    try {
      //TESTING??
      stage = 0;
      estimator.addValues(train1, attrIndex1);
      built = true;

      estimator1 = estimator.makeCopies(getEstimator(), 1)[0];
      
      stage = 1;
      built = false;
      estimator.addValues(train2, attrIndex2);
      built = true;
       
      stage = 2;
      built = false;
      estimator.addValues(train1, attrIndex1);
      built = true;
      
      stage = 3;
      if (!estimator.equals(estimator1)) {
        if (m_Debug) {
          println("\n=== Full report ===\n"
                  + "\nFirst build estimator\n"+
                  estimator.toString() + "\n\n");
          println("\nSecond build estimator\n"+
                  estimator.toString() + "\n\n");
        }
        throw new Exception("Results differ between buildEstimator calls");
      }
      println("yes");
      result[0] = true;
      
      if (false && m_Debug) {
        println("\n=== Full report ===\n"
                + "\nFirst buildEstimator()"
                + "\n\n");
        println("\nSecond buildEstimator()" 
                + "\n\n");
      }
    }
    catch (Exception ex) {
      String msg = ex.getMessage().toLowerCase();
      if (msg.indexOf("worse than zeror") >= 0) {
        println("warning: performs worse than ZeroR");
        result[0] = true;
        result[1] = true;
      } else {
        println("no");
        result[0] = false;
      }
      if (m_Debug) {
        println("\n=== Full Report ===");
        print("Problem during");
        if (built) {
          print(" testing");
        } else {
          print(" training");
        }
        switch (stage) {
          case 0:
            print(" of dataset 1");
            break;
          case 1:
            print(" of dataset 2");
            break;
          case 2:
            print(" of dataset 1 (2nd build)");
            break;
          case 3:
            print(", comparing results from builds of dataset 1");
            break;        
        }
        println(": " + ex.getMessage() + "\n");
        println("here are the datasets:\n");
        println("=== Train1 Dataset ===\n"
            + train1.toString() + "\n");
        println("=== Test1 Dataset ===\n"
            + test1.toString() + "\n\n");
        println("=== Train2 Dataset ===\n"
            + train2.toString() + "\n");
        println("=== Test2 Dataset ===\n"
            + test2.toString() + "\n\n");
      }
    }
    
    return result;
  }
  
  /**
   * Checks basic missing value handling of the scheme. If the missing
   * values cause an exception to be thrown by the scheme, this will be
   * recorded.
   *
   * @param attrTypes attribute types that can be estimated
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @param attributeMissing true if the missing values may be in 
   * the attributes
   * @param classMissing true if the missing values may be in the class
   * @param missingLevel the percentage of missing values
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] canHandleMissing(AttrTypes attrTypes,
                                       int classType,
                                       boolean attributeMissing,
                                       boolean classMissing,
                                       int missingLevel) {
    
    if (missingLevel == 100)
      print("100% ");
    print("missing");
    if (attributeMissing) {
      print(" attribute");
      if (classMissing)
        print(" and");
    }
    if (classMissing)
      print(" class");
    print(" values");
    printAttributeSummary(attrTypes, classType);

    print("...");
    FastVector accepts = new FastVector();
    accepts.addElement("missing");
    accepts.addElement("value");
    accepts.addElement("train");
    int numTrain = getNumInstances(), numTest = getNumInstances(), 
    numClasses = 2;
    
    int numAtts = 1, attrIndex = 0;
    return runBasicTest(attrTypes,
                        numAtts, attrIndex,
                        classType, 
                        missingLevel, attributeMissing, classMissing,
                        numTrain, numTest, numClasses, 
                        accepts);
  }
  
  /**
   * Checks whether an incremental scheme produces the same model when
   * trained incrementally as when batch trained. The model itself
   * cannot be compared, so we compare the evaluation on test data
   * for both models. It is possible to get a false positive on this
   * test (likelihood depends on the estimator).
   *
   * @param attrTypes attribute types that can be estimated
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @return index 0 is true if the test was passed
   */
  protected boolean[] incrementingEquality(AttrTypes attrTypes,
                                           int classType) {
    
    print("incremental training produces the same results"
        + " as batch training");
    printAttributeSummary(attrTypes, classType);

    print("...");
    int numTrain = getNumInstances(), numTest = getNumInstances(), 
    numClasses = 2, missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    
    boolean[] result = new boolean[2];
    Instances train = null;
    Estimator [] estimators = null;
    boolean built = false;
    int attrIndex = 0;
    Vector test;
    try {
      train = makeTestDataset(42, numTrain, 1, attrTypes,
                              numClasses, 
                              classType
                              );

        // prepare training data set and test value list
      test = makeTestValueList(24, numTest, train, attrIndex,
                               attrTypes.getSetType());

      if (missingLevel > 0) {
        addMissing(train, missingLevel, attributeMissing, classMissing, attrIndex);
      }
      estimators = Estimator.makeCopies(getEstimator(), 2);
      estimators[0].addValues(train, attrIndex);
    } catch (Exception ex) {
      throw new Error("Error setting up for tests: " + ex.getMessage());
    }
    try {
      for (int i = 0; i < train.numInstances(); i++) {
        ((IncrementalEstimator)estimators[1]).addValue(train.instance(i).value(attrIndex), 1.0);
      }
      built = true;
      if (!estimators[0].equals(estimators[1])) {
        println("no");
        result[0] = false;
       
        if (m_Debug) {
          println("\n=== Full Report ===");
          println("Results differ between batch and "
              + "incrementally built models.\n"
              + "Depending on the estimator, this may be OK");
          println("Here are the results:\n");
          println("batch built results\n" + estimators[0].toString());
          println("incrementally built results\n" + estimators[1].toString());
          println("Here are the datasets:\n");
          println("=== Train Dataset ===\n"
              + train.toString() + "\n");
          println("=== Test Dataset ===\n"
              + test.toString() + "\n\n");
        }
      }
      else {
        println("yes");
        result[0] = true;
      }
    } catch (Exception ex) {
      result[0] = false;
      
      print("Problem during");
      if (built)
        print(" testing");
      else
        print(" training");
      println(": " + ex.getMessage() + "\n");
    }
    
    return result;
  }
  
  
  /**
   * Checks whether the estimator can handle instance weights.
   * This test compares the estimator performance on two datasets
   * that are identical except for the training weights. If the 
   * results change, then the estimator must be using the weights. It
   * may be possible to get a false positive from this test if the 
   * weight changes aren't significant enough to induce a change
   * in estimator performance (but the weights are chosen to minimize
   * the likelihood of this).
   *
   * @param attrTypes attribute types that can be estimated
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @return index 0 true if the test was passed
   */
  protected boolean[] instanceWeights(AttrTypes attrTypes,
                                      int classType) {
    
    print("estimator uses instance weights");
    printAttributeSummary(attrTypes, classType);

    print("...");

    int numTrain = 2 * getNumInstances(), numTest = getNumInstances(), 
      numClasses = 2, missingLevel = 0;
    boolean attributeMissing = false, classMissing = false;
    
    boolean[] result = new boolean[2];
    Instances train = null;
    Vector test = null;
    Estimator [] estimators = null;
    
    Vector resultProbsO = null;
    Vector resultProbsW = null;
    boolean built = false;
    boolean evalFail = false;
    int attrIndex = 0;
    try {
      train = makeTestDataset(42, numTrain, 1, 
                              attrTypes, numClasses, 
                              classType);
  
      // prepare training data set and test value list
      test = makeTestValueList(24, numTest, train, attrIndex,
                               attrTypes.getSetType());

      if (missingLevel > 0) {
        addMissing(train, missingLevel, attributeMissing, classMissing, attrIndex);
      }

      estimators = Estimator.makeCopies(getEstimator(), 2);

      estimators[0].addValues(train, attrIndex);
      resultProbsO = testWithTestValues(estimators[0], test);

    } catch (Exception ex) {
      throw new Error("Error setting up for tests: " + ex.getMessage());
    }
    try {
            
      // Now modify instance weights and re-built
      for (int i = 0; i < train.numInstances(); i++) {
        train.instance(i).setWeight(0);
      }
      Random random = new Random(1);
      for (int i = 0; i < train.numInstances() / 2; i++) {
        int inst = Math.abs(random.nextInt()) % train.numInstances();
        int weight = Math.abs(random.nextInt()) % 10 + 1;
        train.instance(inst).setWeight(weight);
      }
      estimators[1].addValues(train, attrIndex);
      resultProbsW = testWithTestValues(estimators[1], test);

      built = true;
      if (resultProbsO.equals(resultProbsW)) {
        //      println("no");
        evalFail = true;
        throw new Exception("evalFail");
      }
      
      println("yes");
      result[0] = true;
    } catch (Exception ex) {
      println("no");
      result[0] = false;
      
      if (m_Debug) {
        println("\n=== Full Report ===");
        
        if (evalFail) {
          println("Results don't differ between non-weighted and "
              + "weighted instance models.");
          println("Here are the results:\n");
          println(probsToString(resultProbsO));
        } else {
          print("Problem during");
          if (built) {
            print(" testing");
          } else {
            print(" training");
          }
          println(": " + ex.getMessage() + "\n");
        }
        println("Here are the datasets:\n");
        println("=== Train Dataset ===\n"
            + train.toString() + "\n");
        println("=== Train Weights ===\n");
        for (int i = 0; i < train.numInstances(); i++) {
          println(" " + (i + 1) 
              + "    " + train.instance(i).weight());
        }
        println("=== Test Dataset ===\n"
            + test.toString() + "\n\n");        
        println("(test weights all 1.0\n");
      }
    }
    
    return result;
  }
  
  /**
   * Checks whether the scheme alters the training dataset during
   * training. If the scheme needs to modify the training
   * data it should take a copy of the training data. Currently checks
   * for changes to header structure, number of instances, order of
   * instances, instance weights.
   *
   * @param attrTypes attribute types that can be estimated
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @param attributeMissing true if we know the estimator can handle
   * (at least) moderate missing attribute values
   * @param classMissing true if we know the estimator can handle
   * (at least) moderate missing class values
   * @return index 0 is true if the test was passed
   */
  protected boolean[] datasetIntegrity(AttrTypes attrTypes,
                                       int classType,
                                       boolean attributeMissing,
                                       boolean classMissing) {
    
    Estimator estimator = null;
    print("estimator doesn't alter original datasets");
    printAttributeSummary(attrTypes, classType);
    print("...");
    int numTrain = getNumInstances(), numTest = getNumInstances(), 
    numClasses = 2, missingLevel = 100;
    
    boolean[] result = new boolean[2];
    Instances train = null;
     boolean built = false;
    try {
      train = makeTestDataset(42, numTrain, 1, attrTypes,
                              numClasses, 
                              classType);
      int attrIndex = 0;
 
      if (missingLevel > 0) {
        addMissing(train, missingLevel, attributeMissing, classMissing, attrIndex);
      }
      estimator = Estimator.makeCopies(getEstimator(), 1)[0];
    } catch (Exception ex) {
      throw new Error("Error setting up for tests: " + ex.getMessage());
    }
    try {
      Instances trainCopy = new Instances(train);
      int attrIndex = 0;
      estimator.addValues(trainCopy, attrIndex);
      compareDatasets(train, trainCopy);
      built = true;
      
      println("yes");
      result[0] = true;
    } catch (Exception ex) {
      println("no");
      result[0] = false;
      
      if (m_Debug) {
        println("\n=== Full Report ===");
        print("Problem during");
        if (built) {
          print(" testing");
        } else {
          print(" training");
        }
        println(": " + ex.getMessage() + "\n");
        println("Here are the datasets:\n");
        println("=== Train Dataset ===\n"
            + train.toString() + "\n");
      }
    }
    
    return result;
  }
  
  /**
   * Runs a text on the datasets with the given characteristics.
   * 
   * @param attrTypes attribute types that can be estimated
   * @param numAtts number of attributes
   * @param attrIndex attribute index 
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @param missingLevel the percentage of missing values
   * @param attributeMissing true if the missing values may be in 
   * the attributes
   * @param classMissing true if the missing values may be in the class
   * @param numTrain the number of instances in the training set
   * @param numTest the number of instaces in the test set
   * @param numClasses the number of classes
   * @param accepts the acceptable string in an exception
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] runBasicTest(AttrTypes attrTypes,
                                   int numAtts,
                                   int attrIndex,
                                   int classType,
                                   int missingLevel,
                                   boolean attributeMissing,
                                   boolean classMissing,
                                   int numTrain,
                                   int numTest,
                                   int numClasses,
                                   FastVector accepts) {
    
    return runBasicTest(attrTypes,
                        numAtts,
                        attrIndex,
                        classType, 
                        TestInstances.CLASS_IS_LAST,
                        missingLevel,
                        attributeMissing,
                        classMissing,
                        numTrain,
                        numTest,
                        numClasses,
                accepts);
  }
  
  /**
   * Runs a text on the datasets with the given characteristics.
   * 
   * @param attrTypes attribute types that can be estimated
   * @param numAtts number of attributes
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @param classIndex the attribute index of the class
   * @param missingLevel the percentage of missing values
   * @param attributeMissing true if the missing values may be in 
   * the attributes
   * @param classMissing true if the missing values may be in the class
   * @param numTrain the number of instances in the training set
   * @param numTest the number of instaces in the test set
   * @param numClasses the number of classes
   * @param accepts the acceptable string in an exception
   * @return index 0 is true if the test was passed, index 1 is true if test 
   *         was acceptable
   */
  protected boolean[] runBasicTest(AttrTypes attrTypes,
                                   int numAtts,
                                   int attrIndex,
                                   int classType,
                                   int classIndex,
                                   int missingLevel,
                                   boolean attributeMissing,
                                   boolean classMissing,
                                   int numTrain,
                                   int numTest,
                                   int numClasses,
                                   FastVector accepts) {
    
    boolean[] result = new boolean[2];
    Instances train = null;
    Vector test = null;
    Estimator estimator = null;
    boolean built = false;
   
    try {
      train = makeTestDataset(42, numTrain, numAtts, attrTypes,
          numClasses, 
          classType,
          classIndex);
            
      // prepare training data set and test value list
      if (numTrain > 0) {
        test = makeTestValueList(24, numTest, train, attrIndex,
            attrTypes.getSetType());
     
      } else {
        double min = -10.0;
        double max = 8.0;
        test = makeTestValueList(24, numTest, min, max,
            attrTypes.getSetType());
     }
      
      if (missingLevel > 0) {
        addMissing(train, missingLevel, attributeMissing, classMissing, attrIndex);
      }
      estimator = Estimator.makeCopies(getEstimator(), 1)[0];
    } catch (Exception ex) {
      ex.printStackTrace();
      throw new Error("Error setting up for tests: " + ex.getMessage());
    }
    try {
      estimator.addValues(train, attrIndex);
      built = true;
      
      testWithTestValues(estimator, test);
      
      println("yes");
      result[0] = true;
    } 
    catch (Exception ex) {
      boolean acceptable = false;
      String msg;
      if (ex.getMessage() == null)
        msg = "";
      else
        msg = ex.getMessage().toLowerCase();
      if (msg.indexOf("not in classpath") > -1)
        m_ClasspathProblems = true;
      
      for (int i = 0; i < accepts.size(); i++) {
        if (msg.indexOf((String)accepts.elementAt(i)) >= 0) {
          acceptable = true;
        }
      }
      
      println("no" + (acceptable ? " (OK error message)" : ""));
      result[1] = acceptable;
      
      
      if (m_Debug) {
        println("\n=== Full Report ===");
        print("Problem during");
        if (built) {
          print(" testing");
        } else {
          print(" training");
        }
        println(": " + ex.getMessage() + "\n");
        if (!acceptable) {
          if (accepts.size() > 0) {
            print("Error message doesn't mention ");
            for (int i = 0; i < accepts.size(); i++) {
              if (i != 0) {
                print(" or ");
              }
              print('"' + (String)accepts.elementAt(i) + '"');
            }
          }
          println("here are the datasets:\n");
          println("=== Train Dataset ===\n"
              + train.toString() + "\n");
          println("=== Test Dataset ===\n"
              + test.toString() + "\n\n");
        }
        
      }
    }
    return result;
  }
  
  /**
   * Compare two datasets to see if they differ.
   *
   * @param data1 one set of instances
   * @param data2 the other set of instances
   * @throws Exception if the datasets differ
   */
  protected void compareDatasets(Instances data1, Instances data2)
  throws Exception {
    if (!data2.equalHeaders(data1)) {
      throw new Exception("header has been modified\n" + data2.equalHeadersMsg(data1));
    }
    if (!(data2.numInstances() == data1.numInstances())) {
      throw new Exception("number of instances has changed");
    }
    for (int i = 0; i < data2.numInstances(); i++) {
      Instance orig = data1.instance(i);
      Instance copy = data2.instance(i);
      for (int j = 0; j < orig.numAttributes(); j++) {
        if (orig.isMissing(j)) {
          if (!copy.isMissing(j)) {
            throw new Exception("instances have changed");
          }
        } else if (orig.value(j) != copy.value(j)) {
          throw new Exception("instances have changed");
        }
        if (orig.weight() != copy.weight()) {
          throw new Exception("instance weights have changed");
        }         
      }
    }
  }
  
  /**
   * Add missing values to a dataset.
   *
   * @param data the instances to add missing values to
   * @param level the level of missing values to add (if positive, this
   * is the probability that a value will be set to missing, if negative
   * all but one value will be set to missing (not yet implemented))
   * @param attributeMissing if true, attributes will be modified
   * @param classMissing if true, the class attribute will be modified
   * @param attrIndex index of the attribute
   */
  protected void addMissing(Instances data, int level,
                            boolean attributeMissing, boolean classMissing,
                            int attrIndex) {
    
    int classIndex = data.classIndex();
    Random random = new Random(1);
    for (int i = 0; i < data.numInstances(); i++) {
      Instance current = data.instance(i);

      for (int j = 0; j < data.numAttributes(); j++) {
        if (((j == classIndex) && classMissing) ||
            ((j == attrIndex) && attributeMissing)) {
          if (Math.abs(random.nextInt()) % 100 < level)
            current.setMissing(j);
        }
      }
    }
  }
  
  /**
   * Make a simple set of instances, which can later be modified
   * for use in specific tests.
   *
   * @param seed the random number seed
   * @param numInstances the number of instances to generate
   * @param numAttr the number of attributes
   * @param attrTypes the attribute types
   * @param numClasses the number of classes (if nominal class)
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @return the test dataset
   * @throws Exception if the dataset couldn't be generated
   * @see #process(Instances)
   */
  protected Instances makeTestDataset(int seed, 
                                      int numInstances, 
                                      int numAttr,
                                      AttrTypes attrTypes,
                                      int numClasses, 
                                      int classType)
    throws Exception {
    
    return makeTestDataset(
                           seed,
                           numInstances,
                           numAttr,
                           attrTypes,
                           numClasses, 
                           classType,
                           TestInstances.CLASS_IS_LAST);
  }


  /**
   * Make a simple set of instances with variable position of the class 
   * attribute, which can later be modified for use in specific tests.
   *
   * @param seed the random number seed
   * @param numInstances the number of instances to generate
   * @param numAttr the number of attributes to generate
   * @param attrTypes the type of attrbute that is excepted
   * @param numClasses the number of classes (if nominal class)
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   * @param classIndex the index of the class (0-based, -1 as last)
   * @return the test dataset
   * @throws Exception if the dataset couldn't be generated
   * @see TestInstances#CLASS_IS_LAST
   * @see #process(Instances)
   */
  protected Instances makeTestDataset(int seed, int numInstances, 
                                      int numAttr, AttrTypes attrTypes,
                                      int numClasses, int classType,
                                      int classIndex)
    throws Exception {
    
    TestInstances dataset = new TestInstances();
    
    dataset.setSeed(seed);
    dataset.setNumInstances(numInstances);
    dataset.setNumNominal   (attrTypes.nominal     ? numAttr : 0);
    dataset.setNumNumeric   (attrTypes.numeric     ? numAttr : 0);
    dataset.setNumString    (attrTypes.string      ? numAttr : 0);
    dataset.setNumDate      (attrTypes.date        ? numAttr : 0);
    dataset.setNumRelational(attrTypes.relational  ? numAttr : 0);
    dataset.setNumClasses(numClasses);
    dataset.setClassType(classType);
    dataset.setClassIndex(classIndex);
    
    return process(dataset.generate());
  }

  /**
   * Make a simple set of values. Only one of the num'type' parameters should be larger 0.
   * (just to make parameter similar to the makeTestDataset parameters)
   *
   * @param seed the random number seed
   * @param numValues the number of values to generate
   * @param data the dataset to make test examples for
   * @param attrIndex index of the attribute
   * @param attrType the class type (NUMERIC, NOMINAL, etc.)
   * @throws Exception if the dataset couldn't be generated
   * @see #process(Instances)
   */
  protected Vector makeTestValueList(int seed, int numValues, 
      Instances data, int attrIndex, int attrType)
  throws Exception {
    
    // get min max
    double []minMax = getMinimumMaximum(data, attrIndex);
    double minValue = minMax[0];
    double maxValue = minMax[1];
    
    // make value list and put into a VECTOR
    double range = maxValue - minValue; 
    Vector values = new Vector(numValues); 
    Random random = new Random(seed);
    
    if (attrType == Attribute.NOMINAL) {
      for (int i = 0; i < numValues; i++) {
        Double v = new Double((Math.abs(random.nextInt()) % (int)range)+ (int)minValue);
        values.add(v);
      }
    }
    if (attrType == Attribute.NUMERIC) {
      for (int i = 0; i < numValues; i++) {
        Double v = new Double(random.nextDouble() * range + minValue);
        values.add(v);
      }
    }
    return values;
  }

  /**
   * Make a simple set of values. Only one of the num'type' parameters should be larger 0.
   * (just to make parameter similar to the makeTestDataset parameters)
   *
   * @param seed the random number seed
   * @param numValues the number of values to generate
   * @param minValue the minimal data value
   * @param maxValue the maximal data value
   * @param attrType the class type (NUMERIC, NOMINAL, etc.)
   * @throws Exception if the dataset couldn't be generated
   * @see #process(Instances)
   */
  protected Vector makeTestValueList(int seed, int numValues, 
      double minValue, double maxValue, int attrType)
  throws Exception {
    
      
    // make value list and put into a VECTOR
    double range = maxValue - minValue; 
    Vector values = new Vector(numValues); 
    Random random = new Random(seed);
    
    if (attrType == Attribute.NOMINAL) {
      for (int i = 0; i < numValues; i++) {
        Double v = new Double((Math.abs(random.nextInt()) % (int)range)+ (int)minValue);
        values.add(v);
      }
    }
    if (attrType == Attribute.NUMERIC) {
      for (int i = 0; i < numValues; i++) {
        Double v = new Double(random.nextDouble() * range + minValue);
        values.add(v);
      }
    }
    return values;
  }

  /**
   * Test with test values.
   *
   * @param est estimator to be tested
   * @param test vector with test values
   *
   **/
  protected Vector testWithTestValues(Estimator est, Vector test) {
    
    Vector results = new Vector();
    for (int i = 0; i < test.size(); i++) {
      double testValue = ((Double)(test.elementAt(i))).doubleValue();
      double prob = est.getProbability(testValue);
      Double p = new Double(prob);
      results.add(p);
    }
    return results;
  }

  /**
   * Gets the minimum and maximum of the values a the first attribute
   * of the given data set
   *
   * @param inst the instance
   * @param attrIndex the index of the attribut to find min and max
   * @return the array with the minimum value on index 0 and the max on index 1
   */
  
  protected double[] getMinimumMaximum(Instances inst, int attrIndex) {
    double []minMax = new double[2];
    
    try {
      int num = getMinMax(inst, attrIndex, minMax);
    } catch (Exception ex) {
      ex.printStackTrace();
      System.out.println(ex.getMessage());
    }
    return minMax;
    //      double minValue = minMax[0];
    //      double maxValue = minMax[1];
  }
  
  /** 
   * Find the minimum and the maximum of the attribute and return it in 
   * the last parameter..
   * @param inst instances used to build the estimator
   * @param attrIndex index of the attribute
   * @param minMax the array to return minimum and maximum in
   * @return number of not missing values
   * @exception Exception if parameter minMax wasn't initialized properly
   */
  public static int getMinMax(Instances inst, int attrIndex, double [] minMax) 
    throws Exception {
    double min = Double.NaN;
    double max = Double.NaN;
    Instance instance = null;
    int numNotMissing = 0;
    if ((minMax == null) || (minMax.length < 2)) {
      throw new Exception("Error in Program, privat method getMinMax");
    }
    
    Enumeration enumInst = inst.enumerateInstances();
    if (enumInst.hasMoreElements()) {
      do {
        instance = (Instance) enumInst.nextElement();
      } while (instance.isMissing(attrIndex) && (enumInst.hasMoreElements()));
      
      // add values if not  missing
      if (!instance.isMissing(attrIndex)) {
        numNotMissing++;
        min = instance.value(attrIndex);
        max = instance.value(attrIndex);
      }
      while (enumInst.hasMoreElements()) {
        instance = (Instance) enumInst.nextElement();
        if (!instance.isMissing(attrIndex)) {
          numNotMissing++;
          if (instance.value(attrIndex) < min) {
            min = (instance.value(attrIndex));
          } else {
            if (instance.value(attrIndex) > max) {            
              max = (instance.value(attrIndex));
            }
          }
        }
      }
    }
    minMax[0] = min;
    minMax[1] = max;
    return numNotMissing;
  }

  /**
   * Print the probabilities after testing
   * @param probs vector with probability values
   * @return string with probability values printed
   */ 
  private String probsToString(Vector probs) {
    StringBuffer txt = new StringBuffer (" ");
    for (int i = 0; i < probs.size(); i++) {
      txt.append("" + ((Double)(probs.elementAt(i))).doubleValue() + " ");
    }
    return txt.toString();
  }
  
  /**
   * Provides a hook for derived classes to further modify the data. 
   * 
   * @param data        the data to process
   * @return            the processed data
   * @see #m_PostProcessor
   */
  protected Instances process(Instances data) {
    if (getPostProcessor() == null)
      return data;
    else
      return getPostProcessor().process(data);
  }
  
  /**
   * Print out a short summary string for the dataset characteristics
   *
   * @param attrTypes the attribute types used (NUMERIC, NOMINAL, etc.)
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   */
  protected void printAttributeSummary(AttrTypes attrTypes, int classType) {
    
    String str = "";
    
    if (attrTypes.numeric)
      str += " numeric";
    
    if (attrTypes.nominal) {
      if (str.length() > 0)
        str += " &";
      str += " nominal";
    }
    
    if (attrTypes.string) {
      if (str.length() > 0)
        str += " &";
      str += " string";
    }
    
    if (attrTypes.date) {
      if (str.length() > 0)
        str += " &";
      str += " date";
    }
    
    if (attrTypes.relational) {
      if (str.length() > 0)
        str += " &";
      str += " relational";
    }
    
    str += " attributes)";
    
    switch (classType) {
      case Attribute.NUMERIC:
        str = " (numeric class," + str;
        break;
      case Attribute.NOMINAL:
        str = " (nominal class," + str;
        break;
      case Attribute.STRING:
        str = " (string class," + str;
        break;
      case Attribute.DATE:
        str = " (date class," + str;
        break;
      case Attribute.RELATIONAL:
        str = " (relational class," + str;
        break;
    }
    
    print(str);
  }
  
  /**
   * Print out a short summary string for the dataset characteristics
   *
   * @param attrType the attribute type (NUMERIC, NOMINAL, etc.)
   * @param classType the class type (NUMERIC, NOMINAL, etc.)
   */
  protected void printAttributeSummary(int attrType, int classType) {
    
    String str = "";
    
    switch (attrType) {
    case Attribute.NUMERIC:
      str = " numeric" + str;
      break;
    case Attribute.NOMINAL:
      str = " nominal" + str;
      break;
    case Attribute.STRING:
      str = " string" + str;
      break;
    case Attribute.DATE:
      str = " date" + str;
      break;
    case Attribute.RELATIONAL:
      str = " relational" + str;
      break;
    }
    str += " attribute(s))";
    
    switch (classType) {
    case Attribute.NUMERIC:
      str = " (numeric class," + str;
      break;
    case Attribute.NOMINAL:
      str = " (nominal class," + str;
      break;
    case Attribute.STRING:
      str = " (string class," + str;
      break;
    case Attribute.DATE:
      str = " (date class," + str;
      break;
    case Attribute.RELATIONAL:
      str = " (relational class," + str;
      break;
    }
    
    print(str);
  }
  
  /**
   * Returns the revision string.
   * 
   * @return            the revision
   */
  public String getRevision() {
    return RevisionUtils.extract("$Revision: 4997 $");
  }

  /**
   * Test method for this class
   * 
   * @param args the commandline parameters
   */
  public static void main(String [] args) {
    try {
      CheckEstimator check = new CheckEstimator();
      
      try {
        check.setOptions(args);
        Utils.checkForRemainingOptions(args);
      } catch (Exception ex) {
        String result = ex.getMessage() + "\n\n" + check.getClass().getName().replaceAll(".*\\.", "") + " Options:\n\n";
        Enumeration enu = check.listOptions();
        while (enu.hasMoreElements()) {
          Option option = (Option) enu.nextElement();
          result += option.synopsis() + "\n" + option.description() + "\n";
        }
        throw new Exception(result);
      }
      
      check.doTests();
    } catch (Exception ex) {
      System.err.println(ex.getMessage());
    }
  }
}