source: branches/MetisMQI/src/test/java/weka/classifiers/misc/SerializedClassifierTest.java

/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 * Copyright (C) 2007 University of Waikato, Hamilton, New Zealand
 */

package weka.classifiers.misc;

import weka.classifiers.Classifier;
import weka.classifiers.evaluation.EvaluationUtils;
import weka.core.Attribute;
import weka.core.CheckOptionHandler;
import weka.core.FastVector;
import weka.core.Instances;
import weka.core.SerializationHelper;
import weka.core.TestInstances;
import weka.test.Regression;

import java.io.File;

import junit.framework.Test;
import junit.framework.TestCase;
import junit.framework.TestSuite;

/**
 * Tests SerializedClassifier. Run from the command line with:<p>
 * java weka.classifiers.misc.SerializedClassifierTest
 *
 * @author FracPete (fracpete at waikato dot ac dot nz)
 * @version $Revision: 1.1 $
 */
public class SerializedClassifierTest
  extends TestCase {

  /** the filename for temporary serialized models */
  public final static String MODEL_FILENAME = System.getProperty("user.dir") + "/" + "temp.model";
  
  /** the setup classifier */
  protected SerializedClassifier m_Classifier;
  
  /** the OptionHandler tester */
  protected CheckOptionHandler m_OptionTester;

  /**
   * initializes the test
   * 
   * @param name the name of the test
   */
  public SerializedClassifierTest(String name) {
    super(name);
  }
  
  /**
   * Called by JUnit before each test method.
   *
   * @throws Exception if an error occurs reading the example instances.
   */
  protected void setUp() throws Exception {
    m_Classifier   = null;
    m_OptionTester = new CheckOptionHandler();
    m_OptionTester.setSilent(true);

    // delete temp file
    File file = new File(MODEL_FILENAME);
    if (file.exists())
      file.delete();
  }

  /**
   * Called by JUnit after each test method
   */
  protected void tearDown() {
    m_Classifier   = null;
    m_OptionTester = null;

    // delete temp file
    File file = new File(MODEL_FILENAME);
    if (file.exists())
      file.delete();
  }

  /**
   * creates a classifier, trains and serializes it
   * 
   * @param data        the data to use (J48 with nominal class, M5P with
   *                    numeric class)
   * @return            the results for the data
   */
  protected double[] trainAndSerializeClassifier(Instances data) {
    Classifier  classifier;
    double[]    result;
    int         i;
    
    try {
      // build
      if (data.classAttribute().isNominal())
        classifier = new weka.classifiers.trees.J48();
      else
        classifier = new weka.classifiers.trees.M5P();
      classifier.buildClassifier(data);
      
      // record predictions
      result = new double[data.numInstances()];
      for (i = 0; i < result.length; i++)
        result[i] = classifier.classifyInstance(data.instance(i));
      
      // save
      SerializationHelper.write(MODEL_FILENAME, classifier);
    }
    catch (Exception e) {
      fail("Training base classifier failed: " + e);
      return null;
    }
    
    return result;
  }
  
  /**
   * performs the actual test
   * 
   * @param nomClass    whether to use a nominal class with J48 (TRUE) or 
   *                    a numeric one with M5P (FALSE)
   */
  protected void performTest(boolean nomClass) {
    TestInstances       test;
    Instances           data;
    double[]            originalResults;
    double[]            testResults;
    int                 i;

    // generate data
    try {
      test = new TestInstances();
      if (nomClass) {
        test.setClassType(Attribute.NOMINAL);
        test.setNumNominal(5);
        test.setNumNominalValues(4);
        test.setNumNumeric(0);
      }
      else {
        test.setClassType(Attribute.NUMERIC);
        test.setNumNominal(0);
        test.setNumNumeric(5);
      }
      test.setNumDate(0);
      test.setNumString(0);
      test.setNumRelational(0);
      test.setNumInstances(100);
      data = test.generate();
    }
    catch (Exception e) {
      fail("Generating test data failed: " + e);
      return;
    }
    
    // train and save base classifier
    try {
      originalResults = trainAndSerializeClassifier(data);
    }
    catch (Exception e) {
      fail("Training base classifier failed: " + e);
      return;
    }
    
    // test loading
    try {
      m_Classifier = new SerializedClassifier();
      m_Classifier.setModelFile(new File(MODEL_FILENAME));
      m_Classifier.buildClassifier(data);
    }
    catch (Exception e) {
      fail("Loading/testing of classifier failed: " + e);
    }
    
    // compare results
    try {
      // get results from serialized model
      testResults = new double[data.numInstances()];
      for (i = 0; i < testResults.length; i++)
        testResults[i] = m_Classifier.classifyInstance(data.instance(i));
      
      // compare
      for (i = 0; i < originalResults.length; i++) {
        if (originalResults[i] != testResults[i])
          throw new Exception("Result #" + (i+1) + " differs!");
      }
    }
    catch (Exception e) {
      fail("Comparing results failed: " + e);
    }
  }
  
  /**
   * tests a serialized classifier (J48) handling nominal classes
   */
  public void testNominalClass() {
    performTest(true);
  }
  
  /**
   * tests a serialized classifier (M5P) handling numeric classes
   */
  public void testNumericClass() {
    performTest(true);
  }

  /**
   * Returns a string containing all the predictions.
   *
   * @param predictions a <code>FastVector</code> containing the predictions
   * @return a <code>String</code> representing the vector of predictions.
   */
  protected String predictionsToString(FastVector predictions) {
    StringBuffer sb = new StringBuffer();
    sb.append(predictions.size()).append(" predictions\n");
    for (int i = 0; i < predictions.size(); i++) {
      sb.append(predictions.elementAt(i)).append('\n');
    }
    return sb.toString();
  }

  /**
   * Runs a regression test -- this checks that the output of the tested
   * object matches that in a reference version. When this test is
   * run without any pre-existing reference output, the reference version
   * is created. Uses J48 for this purpose.
   */
  public void testRegression() {
    Regression          reg;
    Instances           train;
    Instances           test;
    Instances           data;
    TestInstances       testInst;
    int                 tot;
    int                 mid;
    EvaluationUtils     evaluation;
    FastVector          regressionResults;
    
    reg = new Regression(this.getClass());

    // generate test data
    try {
      testInst = new TestInstances();
      testInst.setClassType(Attribute.NOMINAL);
      testInst.setNumNominal(5);
      testInst.setNumNominalValues(4);
      testInst.setNumNumeric(0);
      testInst.setNumDate(0);
      testInst.setNumString(0);
      testInst.setNumRelational(0);
      testInst.setNumInstances(100);
      data = testInst.generate();
    }
    catch (Exception e) {
      fail("Failed generating data: " + e);
      return;
    }
    
    // split data into train/test
    tot = data.numInstances();
    mid = tot / 2;
    train = null;
    test = null;
    
    try {
      train = new Instances(data, 0, mid);
      test = new Instances(data, mid, tot - mid);
      m_Classifier = new SerializedClassifier();
      m_Classifier.setModelFile(new File(MODEL_FILENAME));
    } 
    catch (Exception e) {
      e.printStackTrace();
      fail("Problem setting up to use classifier: " + e);
    }

    evaluation = new EvaluationUtils();
    try {
      trainAndSerializeClassifier(train);
      regressionResults = evaluation.getTrainTestPredictions(m_Classifier, train, test);
      reg.println(predictionsToString(regressionResults));
    }
    catch (Exception e) {
      fail("Failed obtaining classifier predictions: " + e);
    }
    
    try {
      String diff = reg.diff();
      if (diff == null) {
        System.err.println("Warning: No reference available, creating."); 
      } else if (!diff.equals("")) {
        fail("Regression test failed. Difference:\n" + diff);
      }
    } 
    catch (java.io.IOException ex) {
      fail("Problem during regression testing.\n" + ex);
    }
  }
  
  /**
   * tests the listing of the options
   */
  public void testListOptions() {
    if (!m_OptionTester.checkListOptions())
      fail("Options cannot be listed via listOptions.");
  }
  
  /**
   * tests the setting of the options
   */
  public void testSetOptions() {
    if (!m_OptionTester.checkSetOptions())
      fail("setOptions method failed.");
  }
  
  /**
   * tests whether there are any remaining options
   */
  public void testRemainingOptions() {
    if (!m_OptionTester.checkRemainingOptions())
      fail("There were 'left-over' options.");
  }
  
  /**
   * tests the whether the user-supplied options stay the same after setting.
   * getting, and re-setting again.
   */
  public void testCanonicalUserOptions() {
    if (!m_OptionTester.checkCanonicalUserOptions())
      fail("setOptions method failed");
  }
  
  /**
   * tests the resetting of the options to the default ones
   */
  public void testResettingOptions() {
    if (!m_OptionTester.checkSetOptions())
      fail("Resetting of options failed");
  }
  
  public static Test suite() {
    return new TestSuite(SerializedClassifierTest.class);
  }

  public static void main(String[] args){
    junit.textui.TestRunner.run(suite());
  }
}