source: src/test/java/weka/clusterers/AbstractClustererTest.java @ 6

/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 * Copyright (C) 2005 University of Waikato, Hamilton, New Zealand
 */

package weka.clusterers;

import weka.core.CheckGOE;
import weka.core.CheckOptionHandler;
import weka.core.Instances;
import weka.core.OptionHandler;
import weka.test.Regression;

import junit.framework.TestCase;

/**
 * Abstract Test class for Clusterers. Internally it uses the class
 * <code>CheckClusterer</code> to determine success or failure of the
 * tests. It follows basically the <code>runTests</code> method.
 *
 * @author FracPete (fracpete at waikato dot ac dot nz)
 * @version $Revision: 1.9 $
 *
 * @see CheckClusterer
 * @see CheckClusterer#runTests(boolean, boolean, boolean)
 */
public abstract class AbstractClustererTest 
  extends TestCase {

  /** The clusterer to be tested */
  protected Clusterer m_Clusterer;

  /** For testing the clusterer */
  protected CheckClusterer m_Tester;
  
  /** whether classifier is updateable */
  protected boolean m_updateableClusterer;

  /** whether clusterer handles weighted instances */
  protected boolean m_weightedInstancesHandler;

  /** whether clusterer handles multi-instance data */
  protected boolean m_multiInstanceHandler;

  /** whether to run CheckClusterer in DEBUG mode */
  protected boolean DEBUG = false;
  
  /** wether clusterer can predict nominal attributes (array index is attribute type of class) */
  protected boolean m_NominalPredictors;
  
  /** wether clusterer can predict numeric attributes (array index is attribute type of class) */
  protected boolean m_NumericPredictors;
  
  /** wether clusterer can predict string attributes (array index is attribute type of class) */
  protected boolean m_StringPredictors;
  
  /** wether clusterer can predict date attributes (array index is attribute type of class) */
  protected boolean m_DatePredictors;
  
  /** wether clusterer can predict relational attributes (array index is attribute type of class) */
  protected boolean m_RelationalPredictors;
  
  /** whether clusterer handles missing values */
  protected boolean m_handleMissingPredictors;
  
  /** the result of the regression test */
  protected String m_RegressionResults;
  
  /** the OptionHandler tester */
  protected CheckOptionHandler m_OptionTester;
  
  /** for testing GOE stuff */
  protected CheckGOE m_GOETester;
  
  /**
   * Constructs the <code>AbstractClustererTest</code>. Called by subclasses.
   *
   * @param name the name of the test class
   */
  public AbstractClustererTest(String name) { 
    super(name); 
  }

  /**
   * Called by JUnit before each test method. This implementation creates
   * the default clusterer to test and loads a test set of Instances.
   *
   * @exception Exception if an error occurs reading the example instances.
   */
  protected void setUp() throws Exception {
    m_Clusterer = getClusterer();
    m_Tester    = new CheckClusterer();
    m_Tester.setSilent(true);
    m_Tester.setClusterer(m_Clusterer);
    m_Tester.setNumInstances(20);
    m_Tester.setDebug(DEBUG);
    m_OptionTester = getOptionTester();
    m_GOETester    = getGOETester();

    m_updateableClusterer         = m_Tester.updateableClusterer()[0];
    m_weightedInstancesHandler     = m_Tester.weightedInstancesHandler()[0];
    m_multiInstanceHandler         = m_Tester.multiInstanceHandler()[0];
    m_NominalPredictors            = false;
    m_NumericPredictors            = false;
    m_StringPredictors             = false;
    m_DatePredictors               = false;
    m_RelationalPredictors         = false;
    m_handleMissingPredictors      = false;
    m_RegressionResults            = "";

    // initialize attributes
    checkAttributes(true,  false, false, false, false, false);
    checkAttributes(false, true,  false, false, false, false);
    checkAttributes(false, false, true,  false, false, false);
    checkAttributes(false, false, false, true,  false, false);
    checkAttributes(false, false, false, false, true,  false);

    // 20% missing values
    m_handleMissingPredictors = checkMissingPredictors(20, false);
  }

  /** Called by JUnit after each test method */
  protected void tearDown() {
    m_Clusterer    = null;
    m_Tester       = null;
    m_OptionTester = null;
    m_GOETester    = null;

    m_updateableClusterer          = false;
    m_weightedInstancesHandler     = false;
    m_NominalPredictors            = false;
    m_NumericPredictors            = false;
    m_StringPredictors             = false;
    m_DatePredictors               = false;
    m_RelationalPredictors         = false;
    m_handleMissingPredictors      = false;
    m_RegressionResults            = "";
  }
  
  /**
   * Configures the CheckOptionHandler uses for testing the optionhandling.
   * Sets the scheme to test.
   * 
   * @return    the fully configured CheckOptionHandler
   */
  protected CheckOptionHandler getOptionTester() {
    CheckOptionHandler          result;
    
    result = new CheckOptionHandler();
    if (getClusterer() instanceof OptionHandler)
      result.setOptionHandler((OptionHandler) getClusterer());
    else
      result.setOptionHandler(null);
    result.setUserOptions(new String[0]);
    result.setSilent(true);
    
    return result;
  }
  
  /**
   * Configures the CheckGOE used for testing GOE stuff.
   * Sets the Clusterer returned from the getClusterer() method.
   * 
   * @return    the fully configured CheckGOE
   * @see       #getClusterer()
   */
  protected CheckGOE getGOETester() {
    CheckGOE            result;
    
    result = new CheckGOE();
    result.setObject(getClusterer());
    result.setSilent(true);
    
    return result;
  }

  /**
   * Used to create an instance of a specific clusterer.
   *
   * @return a suitably configured <code>Clusterer</code> value
   */
  public abstract Clusterer getClusterer();

  /**
   * checks whether at least one attribute type can be handled
   * 
   * @return            true if at least one attribute type can be handled
   */
  protected boolean canPredict() {
    return    m_NominalPredictors
           || m_NumericPredictors
           || m_StringPredictors
           || m_DatePredictors
           || m_RelationalPredictors;
  }

  /**
   * tests whether the clusterer can handle certain attributes and if not,
   * if the exception is OK
   *
   * @param nom         to check for nominal attributes
   * @param num         to check for numeric attributes
   * @param str         to check for string attributes
   * @param dat         to check for date attributes
   * @param rel         to check for relational attributes
   * @param allowFail   whether a junit fail can be executed
   * @see CheckClusterer#canPredict(boolean, boolean, boolean, boolean, boolean, boolean)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  protected void checkAttributes(boolean nom, boolean num, boolean str, 
                                 boolean dat, boolean rel,
                                 boolean allowFail) {
    boolean[]     result;
    String        att;

    // determine text for type of attributes
    att = "";
    if (nom)
      att = "nominal";
    else if (num)
      att = "numeric";
    else if (str)
      att = "string";
    else if (dat)
      att = "date";
    else if (rel)
      att = "relational";
    
    result = m_Tester.canPredict(nom, num, str, dat, rel, m_multiInstanceHandler);
    if (nom)
      m_NominalPredictors = result[0];
    else if (num)
      m_NumericPredictors = result[0];
    else if (str)
      m_StringPredictors = result[0];
    else if (dat)
      m_DatePredictors = result[0];
    else if (rel)
      m_RelationalPredictors = result[0];

    if (!result[0] && !result[1] && allowFail)
      fail("Error handling " + att + " attributes!");
  }

  /**
   * tests whether the clusterer can handle different types of attributes and
   * if not, if the exception is OK
   *
   * @see #checkAttributes(boolean, boolean, boolean, boolean, boolean, boolean)
   */
  public void testAttributes() {
    // nominal
    checkAttributes(true,  false, false, false, false, true);
    // numeric
    checkAttributes(false, true,  false, false, false, true);
    // string
    checkAttributes(false, false, true,  false, false, true);
    // date
    checkAttributes(false, false, false, true,  false, true);
    // relational
    if (!m_multiInstanceHandler)
      checkAttributes(false, false, false, false, true,  true);
  }

  /**
   * tests whether the scheme declares a serialVersionUID.
   */
  public void testSerialVersionUID() {
    boolean[]     result;

    result = m_Tester.declaresSerialVersionUID();

    if (!result[0])
      fail("Doesn't declare serialVersionUID!");
  }

  /**
   * tests whether the clusterer handles instance weights correctly
   *
   * @see CheckClusterer#instanceWeights(boolean, boolean, boolean, boolean, boolean, boolean)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  public void testInstanceWeights() {
    boolean[]     result;
    
    if (m_weightedInstancesHandler) {
      if (!canPredict())
        return;
      
      result = m_Tester.instanceWeights(
          m_NominalPredictors,
          m_NumericPredictors,
          m_StringPredictors,
          m_DatePredictors, 
          m_RelationalPredictors, 
          m_multiInstanceHandler);

      if (!result[0])
        System.err.println("Error handling instance weights!");
    }
  }

  /**
   * tests whether the clusterer can handle zero training instances
   *
   * @see CheckClusterer#canHandleZeroTraining(boolean, boolean, boolean, boolean, boolean, boolean)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  public void testZeroTraining() {
    boolean[]     result;
    
    if (!canPredict())
      return;
    
    result = m_Tester.canHandleZeroTraining(
        m_NominalPredictors, 
        m_NumericPredictors, 
        m_StringPredictors, 
        m_DatePredictors, 
        m_RelationalPredictors, 
        m_multiInstanceHandler);

    if (!result[0] && !result[1])
      fail("Error handling zero training instances!");
  }

  /**
   * checks whether the clusterer can handle the given percentage of
   * missing predictors
   *
   * @param percent     the percentage of missing predictors
   * @param allowFail   if true a fail statement may be executed
   * @return            true if the clusterer can handle it
   */
  protected boolean checkMissingPredictors(int percent, boolean allowFail) {
    boolean[]     result;
    
    result = m_Tester.canHandleMissing(
        m_NominalPredictors, 
        m_NumericPredictors, 
        m_StringPredictors, 
        m_DatePredictors, 
        m_RelationalPredictors, 
        m_multiInstanceHandler, 
        true,
        percent);

    if (allowFail) {
      if (!result[0] && !result[1])
        fail("Error handling " + percent + "% missing predictors!");
    }
    
    return result[0];
  }

  /**
   * tests whether the clusterer can handle missing predictors (20% and 100%)
   *
   * @see CheckClusterer#canHandleMissing(boolean, boolean, boolean, boolean, boolean, boolean, boolean, int)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  public void testMissingPredictors() {
    if (!canPredict())
      return;
    
    // 20% missing
    checkMissingPredictors(20, true);

    // 100% missing
    if (m_handleMissingPredictors)
      checkMissingPredictors(100, true);
  }

  /**
   * tests whether the clusterer correctly initializes in the
   * buildClusterer method
   *
   * @see CheckClusterer#correctBuildInitialisation(boolean, boolean, boolean, boolean, boolean, boolean)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  public void testBuildInitialization() {
    boolean[]     result;
    
    if (!canPredict())
      return;
    
    result = m_Tester.correctBuildInitialisation(
        m_NominalPredictors, 
        m_NumericPredictors, 
        m_StringPredictors, 
        m_DatePredictors, 
        m_RelationalPredictors, 
        m_multiInstanceHandler);

    if (!result[0] && !result[1])
      fail("Incorrect build initialization!");
  }

  /**
   * tests whether the clusterer alters the training set during training.
   *
   * @see CheckClusterer#datasetIntegrity(boolean, boolean, boolean, boolean, boolean, boolean, boolean)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  public void testDatasetIntegrity() {
    boolean[]     result;
  
    if (!canPredict())
      return;
    
    result = m_Tester.datasetIntegrity(
        m_NominalPredictors, 
        m_NumericPredictors, 
        m_StringPredictors, 
        m_DatePredictors, 
        m_RelationalPredictors, 
        m_multiInstanceHandler, 
        m_handleMissingPredictors);

    if (!result[0] && !result[1])
      fail("Training set is altered during training!");
  }

  /**
   * tests whether the classifier produces the same model when trained
   * incrementally as when batch trained.
   *
   * @see CheckClusterer#updatingEquality(boolean, boolean, boolean, boolean, boolean, boolean)
   * @see CheckClusterer#runTests(boolean, boolean, boolean)
   */
  public void testUpdatingEquality() {
    boolean[]     result;
    
    if (m_updateableClusterer) {
        result = m_Tester.updatingEquality(
            m_NominalPredictors, 
            m_NumericPredictors, 
            m_StringPredictors, 
            m_DatePredictors, 
            m_RelationalPredictors, 
            m_multiInstanceHandler);

        if (!result[0])
          System.err.println(
              "Incremental training does not produce same result as batch training!");
    }
  }

  /**
   * Builds a model using the current Clusterer using the given data and 
   * returns the produced cluster assignments.
   * 
   * @param data        the instances to test the Clusterer on
   * @return            a String containing the cluster assignments.
   */
  protected String useClusterer(Instances data) throws Exception {
    String      result;
    Clusterer   clusterer;
    int         i;
    double      cluster;
    
    try {
      clusterer = AbstractClusterer.makeCopy(m_Clusterer);
    } 
    catch (Exception e) {
      clusterer = null;
      e.printStackTrace();
      fail("Problem setting up to use Clusterer: " + e);
    }

    clusterer.buildClusterer(data);
    
    // generate result
    result = "";
    for (i = 0; i < data.numInstances(); i++) {
      if (i > 0)
        result += "\n";
      try {
        cluster = clusterer.clusterInstance(data.instance(i));
        result += "" + (i+1) + ": " + cluster;
      }
      catch (Exception e) {
        result += "" + (i+1) + ": " + e.toString();
      }
    }
    
    return result;
  }

  /**
   * Runs a regression test -- this checks that the output of the tested
   * object matches that in a reference version. When this test is
   * run without any pre-existing reference output, the reference version
   * is created.
   */
  public void testRegression() throws Exception {
    boolean     succeeded;
    Regression  reg;
    Instances   train;
    
    // don't bother if not working correctly
    if (m_Tester.hasClasspathProblems())
      return;
    
    reg       = new Regression(this.getClass());
    train     = null;
    succeeded = false;
    
    train = m_Tester.makeTestDataset(
        42, m_Tester.getNumInstances(), 
        m_NominalPredictors ? 2 : 0,
        m_NumericPredictors ? 1 : 0, 
        m_StringPredictors ? 1 : 0,
        m_DatePredictors ? 1 : 0,
        m_RelationalPredictors ? 1 : 0,
        m_multiInstanceHandler);
    
    try {
      m_RegressionResults = useClusterer(train);
      succeeded = true;
      reg.println(m_RegressionResults);
    }
    catch (Exception e) {
      String msg = e.getMessage().toLowerCase();
      if (msg.indexOf("not in classpath") > -1)
        return;
      
      m_RegressionResults = null;
    }
    
    if (!succeeded) {
      fail("Problem during regression testing: no successful output generated");
    }

    try {
      String diff = reg.diff();
      if (diff == null) {
        System.err.println("Warning: No reference available, creating."); 
      } else if (!diff.equals("")) {
        fail("Regression test failed. Difference:\n" + diff);
      }
    } 
    catch (java.io.IOException ex) {
      fail("Problem during regression testing.\n" + ex);
    }
  }
  
  /**
   * tests the listing of the options
   */
  public void testListOptions() {
    if (m_OptionTester.getOptionHandler() != null) {
      if (!m_OptionTester.checkListOptions())
        fail("Options cannot be listed via listOptions.");
    }
  }
  
  /**
   * tests the setting of the options
   */
  public void testSetOptions() {
    if (m_OptionTester.getOptionHandler() != null) {
      if (!m_OptionTester.checkSetOptions())
        fail("setOptions method failed.");
    }
  }
  
  /**
   * tests whether the default settings are processed correctly
   */
  public void testDefaultOptions() {
    if (m_OptionTester.getOptionHandler() != null) {
      if (!m_OptionTester.checkDefaultOptions())
        fail("Default options were not processed correctly.");
    }
  }
  
  /**
   * tests whether there are any remaining options
   */
  public void testRemainingOptions() {
    if (m_OptionTester.getOptionHandler() != null) {
      if (!m_OptionTester.checkRemainingOptions())
        fail("There were 'left-over' options.");
    }
  }
  
  /**
   * tests the whether the user-supplied options stay the same after setting.
   * getting, and re-setting again.
   * 
   * @see       #getOptionTester()
   */
  public void testCanonicalUserOptions() {
    if (m_OptionTester.getOptionHandler() != null) {
      if (!m_OptionTester.checkCanonicalUserOptions())
        fail("setOptions method failed");
    }
  }
  
  /**
   * tests the resetting of the options to the default ones
   */
  public void testResettingOptions() {
    if (m_OptionTester.getOptionHandler() != null) {
      if (!m_OptionTester.checkSetOptions())
        fail("Resetting of options failed");
    }
  }
  
  /**
   * tests for a globalInfo method
   */
  public void testGlobalInfo() {
    if (!m_GOETester.checkGlobalInfo())
      fail("No globalInfo method");
  }
  
  /**
   * tests the tool tips
   */
  public void testToolTips() {
    if (!m_GOETester.checkToolTips())
      fail("Tool tips inconsistent");
  }
}