source: branches/MetisMQI/src/main/java/weka/filters/unsupervised/attribute/NumericCleaner.java

/*
 *    This program is free software; you can redistribute it and/or modify
 *    it under the terms of the GNU General Public License as published by
 *    the Free Software Foundation; either version 2 of the License, or
 *    (at your option) any later version.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU General Public License for more details.
 *
 *    You should have received a copy of the GNU General Public License
 *    along with this program; if not, write to the Free Software
 *    Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 */

/*
 * NumericCleaner.java
 * Copyright (C) 2006 University of Waikato, Hamilton, New Zealand
 */

package weka.filters.unsupervised.attribute;

import weka.core.Capabilities;
import weka.core.Instance; 
import weka.core.DenseInstance;
import weka.core.Instances;
import weka.core.Option;
import weka.core.Range;
import weka.core.RevisionUtils;
import weka.core.Utils;
import weka.core.Capabilities.Capability;
import weka.filters.SimpleStreamFilter;

import java.util.Enumeration;
import java.util.Vector;


/**
 <!-- globalinfo-start -->
 * A filter that 'cleanses' the numeric data from values that are too small, too big or very close to a certain value (e.g., 0) and sets these values to a pre-defined default.
 * <p/>
 <!-- globalinfo-end -->
 * 
 <!-- options-start -->
 * Valid options are: <p/>
 * 
 * <pre> -D
 *  Turns on output of debugging information.</pre>
 * 
 * <pre> -min &lt;double&gt;
 *  The minimum threshold. (default -Double.MAX_VALUE)</pre>
 * 
 * <pre> -min-default &lt;double&gt;
 *  The replacement for values smaller than the minimum threshold.
 *  (default -Double.MAX_VALUE)</pre>
 * 
 * <pre> -max &lt;double&gt;
 *  The maximum threshold. (default Double.MAX_VALUE)</pre>
 * 
 * <pre> -max-default &lt;double&gt;
 *  The replacement for values larger than the maximum threshold.
 *  (default Double.MAX_VALUE)</pre>
 * 
 * <pre> -closeto &lt;double&gt;
 *  The number values are checked for closeness. (default 0)</pre>
 * 
 * <pre> -closeto-default &lt;double&gt;
 *  The replacement for values that are close to '-closeto'.
 *  (default 0)</pre>
 * 
 * <pre> -closeto-tolerance &lt;double&gt;
 *  The tolerance below which numbers are considered being close to 
 *  to each other. (default 1E-6)</pre>
 * 
 * <pre> -decimals &lt;int&gt;
 *  The number of decimals to round to, -1 means no rounding at all.
 *  (default -1)</pre>
 * 
 * <pre> -R &lt;col1,col2,...&gt;
 *  The list of columns to cleanse, e.g., first-last or first-3,5-last.
 *  (default first-last)</pre>
 * 
 * <pre> -V
 *  Inverts the matching sense.</pre>
 * 
 * <pre> -include-class
 *  Whether to include the class in the cleansing.
 *  The class column will always be skipped, if this flag is not
 *  present. (default no)</pre>
 * 
 <!-- options-end -->
 *
 * @author  fracpete (fracpete at waikato dot ac dot nz)
 * @version $Revision: 5987 $
 */
public class NumericCleaner
  extends SimpleStreamFilter {

  /** for serialization */
  private static final long serialVersionUID = -352890679895066592L;

  /** the minimum threshold */
  protected double m_MinThreshold = -Double.MAX_VALUE;

  /** the minimum default replacement value */
  protected double m_MinDefault = -Double.MAX_VALUE;

  /** the maximum threshold */
  protected double m_MaxThreshold = Double.MAX_VALUE;

  /** the maximum default replacement value */
  protected double m_MaxDefault = Double.MAX_VALUE;

  /** the number the values are checked for closeness to */
  protected double m_CloseTo = 0;

  /** the default replacement value for numbers "close-to" */
  protected double m_CloseToDefault = 0;

  /** the tolerance distance, below which numbers are considered being "close-to" */
  protected double m_CloseToTolerance = 1E-6;

  /** Stores which columns to cleanse */
  protected Range m_Cols = new Range("first-last");

  /** whether to include the class attribute */
  protected boolean m_IncludeClass = false;
  
  /** the number of decimals to round to (-1 means no rounding) */
  protected int m_Decimals = -1;
  
  /**
   * Returns a string describing this filter.
   *
   * @return      a description of the filter suitable for
   *              displaying in the explorer/experimenter gui
   */
  public String globalInfo() {
    return 
        "A filter that 'cleanses' the numeric data from values that are too "
      + "small, too big or very close to a certain value (e.g., 0) and sets "
      + "these values to a pre-defined default.";
  }

  /**
   * Returns an enumeration describing the available options.
   *
   * @return an enumeration of all the available options.
   */
  public Enumeration listOptions() {
    Vector        result;
    Enumeration   enm;

    result = new Vector();

    enm = super.listOptions();
    while (enm.hasMoreElements())
      result.addElement(enm.nextElement());

    result.addElement(new Option(
        "\tThe minimum threshold. (default -Double.MAX_VALUE)",
        "min", 1, "-min <double>"));
    
    result.addElement(new Option(
        "\tThe replacement for values smaller than the minimum threshold.\n"
        + "\t(default -Double.MAX_VALUE)",
        "min-default", 1, "-min-default <double>"));

    result.addElement(new Option(
        "\tThe maximum threshold. (default Double.MAX_VALUE)",
        "max", 1, "-max <double>"));
    
    result.addElement(new Option(
        "\tThe replacement for values larger than the maximum threshold.\n"
        + "\t(default Double.MAX_VALUE)",
        "max-default", 1, "-max-default <double>"));

    result.addElement(new Option(
        "\tThe number values are checked for closeness. (default 0)",
        "closeto", 1, "-closeto <double>"));
    
    result.addElement(new Option(
        "\tThe replacement for values that are close to '-closeto'.\n"
        + "\t(default 0)",
        "closeto-default", 1, "-closeto-default <double>"));
    
    result.addElement(new Option(
        "\tThe tolerance below which numbers are considered being close to \n"
        + "\tto each other. (default 1E-6)",
        "closeto-tolerance", 1, "-closeto-tolerance <double>"));

    result.addElement(new Option(
        "\tThe number of decimals to round to, -1 means no rounding at all.\n"
        + "\t(default -1)",
        "decimals", 1, "-decimals <int>"));
    
    result.addElement(new Option(
        "\tThe list of columns to cleanse, e.g., first-last or first-3,5-last.\n"
        + "\t(default first-last)",
        "R", 1, "-R <col1,col2,...>"));

    result.addElement(new Option(
        "\tInverts the matching sense.",
        "V", 0, "-V"));

    result.addElement(new Option(
        "\tWhether to include the class in the cleansing.\n"
        + "\tThe class column will always be skipped, if this flag is not\n"
        + "\tpresent. (default no)",
        "include-class", 0, "-include-class"));

    return result.elements();
  }       

  /**
   * Gets the current settings of the filter.
   *
   * @return an array of strings suitable for passing to setOptions
   */
  public String[] getOptions() {
    int       i;
    Vector    result;
    String[]  options;

    result = new Vector();
    options = super.getOptions();
    for (i = 0; i < options.length; i++)
      result.add(options[i]);

    result.add("-min"); 
    result.add("" + m_MinThreshold);

    result.add("-min-default"); 
    result.add("" + m_MinDefault);

    result.add("-max"); 
    result.add("" + m_MaxThreshold);

    result.add("-max-default"); 
    result.add("" + m_MaxDefault);

    result.add("-closeto"); 
    result.add("" + m_CloseTo);

    result.add("-closeto-default"); 
    result.add("" + m_CloseToDefault);
    
    result.add("-closeto-tolerance"); 
    result.add("" + m_CloseToTolerance);

    result.add("-R"); 
    result.add("" + m_Cols.getRanges());

    if (m_Cols.getInvert())
      result.add("-V");
    
    if (m_IncludeClass)
      result.add("-include-class"); 

    result.add("-decimals"); 
    result.add("" + getDecimals());

    return (String[]) result.toArray(new String[result.size()]);          
  }       

  /**
   * Parses a given list of options. <p/>
   *
   <!-- options-start -->
   * Valid options are: <p/>
   * 
   * <pre> -D
   *  Turns on output of debugging information.</pre>
   * 
   * <pre> -min &lt;double&gt;
   *  The minimum threshold. (default -Double.MAX_VALUE)</pre>
   * 
   * <pre> -min-default &lt;double&gt;
   *  The replacement for values smaller than the minimum threshold.
   *  (default -Double.MAX_VALUE)</pre>
   * 
   * <pre> -max &lt;double&gt;
   *  The maximum threshold. (default Double.MAX_VALUE)</pre>
   * 
   * <pre> -max-default &lt;double&gt;
   *  The replacement for values larger than the maximum threshold.
   *  (default Double.MAX_VALUE)</pre>
   * 
   * <pre> -closeto &lt;double&gt;
   *  The number values are checked for closeness. (default 0)</pre>
   * 
   * <pre> -closeto-default &lt;double&gt;
   *  The replacement for values that are close to '-closeto'.
   *  (default 0)</pre>
   * 
   * <pre> -closeto-tolerance &lt;double&gt;
   *  The tolerance below which numbers are considered being close to 
   *  to each other. (default 1E-6)</pre>
   * 
   * <pre> -decimals &lt;int&gt;
   *  The number of decimals to round to, -1 means no rounding at all.
   *  (default -1)</pre>
   * 
   * <pre> -R &lt;col1,col2,...&gt;
   *  The list of columns to cleanse, e.g., first-last or first-3,5-last.
   *  (default first-last)</pre>
   * 
   * <pre> -V
   *  Inverts the matching sense.</pre>
   * 
   * <pre> -include-class
   *  Whether to include the class in the cleansing.
   *  The class column will always be skipped, if this flag is not
   *  present. (default no)</pre>
   * 
   <!-- options-end -->
   * 
   * @param options the list of options as an array of strings
   * @throws Exception if an option is not supported 
   */
  public void setOptions(String[] options) throws Exception {
    String      tmpStr;

    tmpStr = Utils.getOption("min", options);
    if (tmpStr.length() != 0)
      setMinThreshold(Double.parseDouble(tmpStr));
    else
      setMinThreshold(-Double.MAX_VALUE);
    
    tmpStr = Utils.getOption("min-default", options);
    if (tmpStr.length() != 0)
      setMinDefault(Double.parseDouble(tmpStr));
    else
      setMinDefault(-Double.MAX_VALUE);
    
    tmpStr = Utils.getOption("max", options);
    if (tmpStr.length() != 0)
      setMaxThreshold(Double.parseDouble(tmpStr));
    else
      setMaxThreshold(Double.MAX_VALUE);
    
    tmpStr = Utils.getOption("max-default", options);
    if (tmpStr.length() != 0)
      setMaxDefault(Double.parseDouble(tmpStr));
    else
      setMaxDefault(Double.MAX_VALUE);
    
    tmpStr = Utils.getOption("closeto", options);
    if (tmpStr.length() != 0)
      setCloseTo(Double.parseDouble(tmpStr));
    else
      setCloseTo(0);
    
    tmpStr = Utils.getOption("closeto-default", options);
    if (tmpStr.length() != 0)
      setCloseToDefault(Double.parseDouble(tmpStr));
    else
      setCloseToDefault(0);
    
    tmpStr = Utils.getOption("closeto-tolerance", options);
    if (tmpStr.length() != 0)
      setCloseToTolerance(Double.parseDouble(tmpStr));
    else
      setCloseToTolerance(1E-6);
    
    tmpStr = Utils.getOption("R", options);
    if (tmpStr.length() != 0)
      setAttributeIndices(tmpStr);
    else
      setAttributeIndices("first-last");
    
    setInvertSelection(Utils.getFlag("V", options));
    
    setIncludeClass(Utils.getFlag("include-class", options));

    tmpStr = Utils.getOption("decimals", options);
    if (tmpStr.length() != 0)
      setDecimals(Integer.parseInt(tmpStr));
    else
      setDecimals(-1);
    
    super.setOptions(options);
  }       

  /** 
   * Returns the Capabilities of this filter.
   *
   * @return            the capabilities of this object
   * @see               Capabilities
   */
  public Capabilities getCapabilities() {
    Capabilities result = super.getCapabilities();
    result.disableAll();

    // attributes
    result.enableAllAttributes();
    result.enable(Capability.MISSING_VALUES);
    
    // class
    result.enableAllClasses();
    result.enable(Capability.MISSING_CLASS_VALUES);
    result.enable(Capability.NO_CLASS);
    
    return result;
  }
  
  /**
   * Determines the output format based on the input format and returns 
   * this. In case the output format cannot be returned immediately, i.e.,
   * immediateOutputFormat() returns false, then this method will be called
   * from batchFinished().
   *
   * @param inputFormat     the input format to base the output format on
   * @return                the output format
   * @throws Exception      in case the determination goes wrong
   * @see   #hasImmediateOutputFormat()
   * @see   #batchFinished()
   */
  protected Instances determineOutputFormat(Instances inputFormat)
      throws Exception {

    m_Cols.setUpper(inputFormat.numAttributes() - 1);
    
    return new Instances(inputFormat);
  }

  /**
   * processes the given instance (may change the provided instance) and
   * returns the modified version.
   *
   * @param instance    the instance to process
   * @return            the modified data
   * @throws Exception  in case the processing goes wrong
   */
  protected Instance process(Instance instance) throws Exception {
    Instance            result;
    int                 i;
    double              val;
    double              factor;
    
    result = (Instance) instance.copy();
    
    if (m_Decimals > -1)
      factor = StrictMath.pow(10, m_Decimals);
    else
      factor = 1;
    
    for (i = 0; i < result.numAttributes(); i++) {
      // only numeric attributes
      if (!result.attribute(i).isNumeric())
        continue;

      // out of range?
      if (!m_Cols.isInRange(i))
        continue;
      
      // skip class?
      if ( (result.classIndex() == i) && (!m_IncludeClass) )
        continue;
      
      // too small?
      if (result.value(i) < m_MinThreshold) {
        if (getDebug())
          System.out.println("Too small: " + result.value(i) + " -> " + m_MinDefault);
        result.setValue(i, m_MinDefault);
      }
      // too big?
      else if (result.value(i) > m_MaxThreshold) {
        if (getDebug())
          System.out.println("Too big: " + result.value(i) + " -> " + m_MaxDefault);
        result.setValue(i, m_MaxDefault);
      }
      // too close?
      else if (    (result.value(i) - m_CloseTo < m_CloseToTolerance) 
                && (m_CloseTo - result.value(i) < m_CloseToTolerance) 
                && (result.value(i) != m_CloseTo) ) {
        if (getDebug())
          System.out.println("Too close: " + result.value(i) + " -> " + m_CloseToDefault);
        result.setValue(i, m_CloseToDefault);
      }
      
      // decimals?
      if (m_Decimals > -1) {
        val = result.value(i);
        val = StrictMath.round(val * factor) / factor;
        result.setValue(i, val);
      }
    }

    return result;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String minThresholdTipText() {
    return "The minimum threshold below values are replaced by a default.";
  }

  /**
   * Get the minimum threshold. 
   *
   * @return            the minimum threshold.
   */
  public double getMinThreshold() {
    return m_MinThreshold;
  }

  /**
   * Set the minimum threshold. 
   *
   * @param value       the minimum threshold to use.
   */
  public void setMinThreshold(double value) {
    m_MinThreshold = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String minDefaultTipText() {
    return "The default value to replace values that are below the minimum threshold.";
  }

  /**
   * Get the minimum default. 
   *
   * @return            the minimum default.
   */
  public double getMinDefault() {
    return m_MinDefault;
  }

  /**
   * Set the minimum default. 
   *
   * @param value       the minimum default to use.
   */
  public void setMinDefault(double value) {
    m_MinDefault = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String maxThresholdTipText() {
    return "The maximum threshold above values are replaced by a default.";
  }

  /**
   * Get the maximum threshold. 
   *
   * @return            the maximum threshold.
   */
  public double getMaxThreshold() {
    return m_MaxThreshold;
  }

  /**
   * Set the maximum threshold. 
   *
   * @param value       the maximum threshold to use.
   */
  public void setMaxThreshold(double value) {
    m_MaxThreshold = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String maxDefaultTipText() {
    return "The default value to replace values that are above the maximum threshold.";
  }

  /**
   * Get the maximum default. 
   *
   * @return            the maximum default.
   */
  public double getMaxDefault() {
    return m_MaxDefault;
  }

  /**
   * Set the naximum default. 
   *
   * @param value       the maximum default to use.
   */
  public void setMaxDefault(double value) {
    m_MaxDefault = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String closeToTipText() {
    return 
        "The number values are checked for whether they are too close to "
      + "and get replaced by a default.";
  }

  /**
   * Get the "close to" number.
   *
   * @return            the "close to" number.
   */
  public double getCloseTo() {
    return m_CloseTo;
  }

  /**
   * Set the "close to" number.
   *
   * @param value       the number to use for checking closeness.
   */
  public void setCloseTo(double value) {
    m_CloseTo = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String closeToDefaultTipText() {
    return "The default value to replace values with that are too close.";
  }

  /**
   * Get the "close to" default.
   *
   * @return            the "close to" default.
   */
  public double getCloseToDefault() {
    return m_CloseToDefault;
  }

  /**
   * Set the "close to" default. 
   *
   * @param value       the "close to" default to use.
   */
  public void setCloseToDefault(double value) {
    m_CloseToDefault = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String closeToToleranceTipText() {
    return "The value below which values are considered close to.";
  }

  /**
   * Get the "close to" Tolerance.
   *
   * @return            the "close to" Tolerance.
   */
  public double getCloseToTolerance() {
    return m_CloseToTolerance;
  }

  /**
   * Set the "close to" Tolerance. 
   *
   * @param value       the "close to" Tolerance to use.
   */
  public void setCloseToTolerance(double value) {
    m_CloseToTolerance = value;
  }

  /**
   * Returns the tip text for this property
   *
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String attributeIndicesTipText() {
    return "The selection of columns to use in the cleansing processs, first and last are valid indices.";
  }

  /**
   * Gets the selection of the columns, e.g., first-last or first-3,5-last
   *
   * @return            the selected indices
   */
  public String getAttributeIndices() {
    return m_Cols.getRanges();
  }

  /**
   * Sets the columns to use, e.g., first-last or first-3,5-last
   *
   * @param value       the columns to use
   */
  public void setAttributeIndices(String value) {
    m_Cols.setRanges(value);
  }

  /**
   * Returns the tip text for this property
   *
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String invertSelectionTipText() {
    return "If enabled the selection of the columns is inverted.";
  }

  /**
   * Gets whether the selection of the columns is inverted
   *
   * @return            true if the selection is inverted
   */
  public boolean getInvertSelection() {
    return m_Cols.getInvert();
  }

  /**
   * Sets whether the selection of the indices is inverted or not
   *
   * @param value       the new invert setting
   */
  public void setInvertSelection(boolean value) {
    m_Cols.setInvert(value);
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String includeClassTipText() {
    return "If disabled, the class attribute will be always left out of the cleaning process.";
  }

  /**
   * Gets whether the class is included in the cleaning process or always 
   * skipped.
   *
   * @return            true if the class can be considered for cleaning.
   */
  public boolean getIncludeClass() {
    return m_IncludeClass;
  }

  /**
   * Sets whether the class can be cleaned, too.
   *
   * @param value       true if the class can be cleansed, too
   */
  public void setIncludeClass(boolean value) {
    m_IncludeClass = value;
  }

  /**
   * Returns the tip text for this property
   * 
   * @return            tip text for this property suitable for
   *                    displaying in the explorer/experimenter gui
   */
  public String decimalsTipText() {
    return "The number of decimals to round to, -1 means no rounding at all.";
  }

  /**
   * Get the number of decimals to round to. 
   *
   * @return            the number of decimals.
   */
  public int getDecimals() {
    return m_Decimals;
  }

  /**
   * Set the number of decimals to round to.
   *
   * @param value       the number of decimals.
   */
  public void setDecimals(int value) {
    m_Decimals = value;
  }
  
  /**
   * Returns the revision string.
   * 
   * @return            the revision
   */
  public String getRevision() {
    return RevisionUtils.extract("$Revision: 5987 $");
  }

  /**
   * Runs the filter from commandline, use "-h" to see all options.
   * 
   * @param args the commandline options for the filter
   */
  public static void main(String[] args) {
    runFilter(new NumericCleaner(), args);
  }
}