1 | /* |
---|
2 | * This program is free software; you can redistribute it and/or modify |
---|
3 | * it under the terms of the GNU General Public License as published by |
---|
4 | * the Free Software Foundation; either version 2 of the License, or |
---|
5 | * (at your option) any later version. |
---|
6 | * |
---|
7 | * This program is distributed in the hope that it will be useful, |
---|
8 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
9 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
10 | * GNU General Public License for more details. |
---|
11 | * |
---|
12 | * You should have received a copy of the GNU General Public License |
---|
13 | * along with this program; if not, write to the Free Software |
---|
14 | * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. |
---|
15 | */ |
---|
16 | |
---|
17 | /* |
---|
18 | * Copyright (C) 2006 University of Waikato |
---|
19 | */ |
---|
20 | |
---|
21 | package weka.associations; |
---|
22 | |
---|
23 | import weka.core.Attribute; |
---|
24 | import weka.core.CheckGOE; |
---|
25 | import weka.core.CheckOptionHandler; |
---|
26 | import weka.core.Instances; |
---|
27 | import weka.core.OptionHandler; |
---|
28 | import weka.core.CheckScheme.PostProcessor; |
---|
29 | import weka.test.Regression; |
---|
30 | |
---|
31 | import junit.framework.TestCase; |
---|
32 | |
---|
33 | /** |
---|
34 | * Abstract Test class for Associators. Internally it uses the class |
---|
35 | * <code>CheckAssociator</code> to determine success or failure of the |
---|
36 | * tests. It follows basically the <code>testsPerClassType</code> method. |
---|
37 | * |
---|
38 | * @author FracPete (fracpete at waikato dot ac dot nz) |
---|
39 | * @version $Revision: 1.7 $ |
---|
40 | * |
---|
41 | * @see CheckAssociator |
---|
42 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
43 | * @see PostProcessor |
---|
44 | */ |
---|
45 | public abstract class AbstractAssociatorTest |
---|
46 | extends TestCase { |
---|
47 | |
---|
48 | /** The Associator to be tested */ |
---|
49 | protected Associator m_Associator; |
---|
50 | |
---|
51 | /** For testing the Associator */ |
---|
52 | protected CheckAssociator m_Tester; |
---|
53 | |
---|
54 | /** whether Associator handles weighted instances */ |
---|
55 | protected boolean m_weightedInstancesHandler; |
---|
56 | |
---|
57 | /** whether Associator handles multi-instance data */ |
---|
58 | protected boolean m_multiInstanceHandler; |
---|
59 | |
---|
60 | /** the number of classes to test with testNClasses() |
---|
61 | * @see #testNClasses() */ |
---|
62 | protected int m_NClasses; |
---|
63 | |
---|
64 | /** whether to run CheckAssociator in DEBUG mode */ |
---|
65 | protected boolean DEBUG = false; |
---|
66 | |
---|
67 | /** the attribute type with the lowest value */ |
---|
68 | protected final static int FIRST_CLASSTYPE = Attribute.NUMERIC; |
---|
69 | |
---|
70 | /** the attribute type with the highest value */ |
---|
71 | protected final static int LAST_CLASSTYPE = Attribute.RELATIONAL; |
---|
72 | |
---|
73 | /** wether Associator can predict nominal attributes (array index is attribute type of class) */ |
---|
74 | protected boolean[] m_NominalPredictors; |
---|
75 | |
---|
76 | /** wether Associator can predict numeric attributes (array index is attribute type of class) */ |
---|
77 | protected boolean[] m_NumericPredictors; |
---|
78 | |
---|
79 | /** wether Associator can predict string attributes (array index is attribute type of class) */ |
---|
80 | protected boolean[] m_StringPredictors; |
---|
81 | |
---|
82 | /** wether Associator can predict date attributes (array index is attribute type of class) */ |
---|
83 | protected boolean[] m_DatePredictors; |
---|
84 | |
---|
85 | /** wether Associator can predict relational attributes (array index is attribute type of class) */ |
---|
86 | protected boolean[] m_RelationalPredictors; |
---|
87 | |
---|
88 | /** whether Associator handles missing values */ |
---|
89 | protected boolean[] m_handleMissingPredictors; |
---|
90 | |
---|
91 | /** whether Associator handles class with only missing values */ |
---|
92 | protected boolean[] m_handleMissingClass; |
---|
93 | |
---|
94 | /** the results of the regression tests */ |
---|
95 | protected String[] m_RegressionResults; |
---|
96 | |
---|
97 | /** the OptionHandler tester */ |
---|
98 | protected CheckOptionHandler m_OptionTester; |
---|
99 | |
---|
100 | /** for testing GOE stuff */ |
---|
101 | protected CheckGOE m_GOETester; |
---|
102 | |
---|
103 | /** |
---|
104 | * Constructs the <code>AbstractAssociatorTest</code>. Called by subclasses. |
---|
105 | * |
---|
106 | * @param name the name of the test class |
---|
107 | */ |
---|
108 | public AbstractAssociatorTest(String name) { |
---|
109 | super(name); |
---|
110 | } |
---|
111 | |
---|
112 | /** |
---|
113 | * returns a custom PostProcessor for the CheckClassifier datasets, currently |
---|
114 | * only null. |
---|
115 | * |
---|
116 | * @return a custom PostProcessor, if necessary |
---|
117 | * @see PostProcessor |
---|
118 | */ |
---|
119 | protected PostProcessor getPostProcessor() { |
---|
120 | return null; |
---|
121 | } |
---|
122 | |
---|
123 | /** |
---|
124 | * configures the CheckAssociator instance used throughout the tests |
---|
125 | * |
---|
126 | * @return the fully configured CheckAssociator instance used for testing |
---|
127 | */ |
---|
128 | protected CheckAssociator getTester() { |
---|
129 | CheckAssociator result; |
---|
130 | |
---|
131 | result = new CheckAssociator(); |
---|
132 | result.setSilent(true); |
---|
133 | result.setAssociator(m_Associator); |
---|
134 | result.setNumInstances(20); |
---|
135 | result.setDebug(DEBUG); |
---|
136 | result.setPostProcessor(getPostProcessor()); |
---|
137 | |
---|
138 | return result; |
---|
139 | } |
---|
140 | |
---|
141 | /** |
---|
142 | * Configures the CheckOptionHandler used for testing the optionhandling. |
---|
143 | * Sets the Associator returned from the getAssociator() method. |
---|
144 | * |
---|
145 | * @return the fully configured CheckOptionHandler |
---|
146 | * @see #getAssociator() |
---|
147 | */ |
---|
148 | protected CheckOptionHandler getOptionTester() { |
---|
149 | CheckOptionHandler result; |
---|
150 | |
---|
151 | result = new CheckOptionHandler(); |
---|
152 | if (getAssociator() instanceof OptionHandler) |
---|
153 | result.setOptionHandler((OptionHandler) getAssociator()); |
---|
154 | else |
---|
155 | result.setOptionHandler(null); |
---|
156 | result.setUserOptions(new String[0]); |
---|
157 | result.setSilent(true); |
---|
158 | |
---|
159 | return result; |
---|
160 | } |
---|
161 | |
---|
162 | /** |
---|
163 | * Configures the CheckGOE used for testing GOE stuff. |
---|
164 | * Sets the Associator returned from the getAssociator() method. |
---|
165 | * |
---|
166 | * @return the fully configured CheckGOE |
---|
167 | * @see #getAssociator() |
---|
168 | */ |
---|
169 | protected CheckGOE getGOETester() { |
---|
170 | CheckGOE result; |
---|
171 | |
---|
172 | result = new CheckGOE(); |
---|
173 | result.setObject(getAssociator()); |
---|
174 | result.setSilent(true); |
---|
175 | |
---|
176 | return result; |
---|
177 | } |
---|
178 | |
---|
179 | /** |
---|
180 | * Called by JUnit before each test method. This implementation creates |
---|
181 | * the default Associator to test and loads a test set of Instances. |
---|
182 | * |
---|
183 | * @exception Exception if an error occurs reading the example instances. |
---|
184 | */ |
---|
185 | protected void setUp() throws Exception { |
---|
186 | m_Associator = getAssociator(); |
---|
187 | m_Tester = getTester(); |
---|
188 | m_OptionTester = getOptionTester(); |
---|
189 | m_GOETester = getGOETester(); |
---|
190 | |
---|
191 | m_weightedInstancesHandler = m_Tester.weightedInstancesHandler()[0]; |
---|
192 | m_multiInstanceHandler = m_Tester.multiInstanceHandler()[0]; |
---|
193 | // LAST_CLASSTYPE+1 = no class attribute |
---|
194 | m_NominalPredictors = new boolean[LAST_CLASSTYPE + 2]; |
---|
195 | m_NumericPredictors = new boolean[LAST_CLASSTYPE + 2]; |
---|
196 | m_StringPredictors = new boolean[LAST_CLASSTYPE + 2]; |
---|
197 | m_DatePredictors = new boolean[LAST_CLASSTYPE + 2]; |
---|
198 | m_RelationalPredictors = new boolean[LAST_CLASSTYPE + 2]; |
---|
199 | m_handleMissingPredictors = new boolean[LAST_CLASSTYPE + 2]; |
---|
200 | m_handleMissingClass = new boolean[LAST_CLASSTYPE + 2]; |
---|
201 | m_RegressionResults = new String[LAST_CLASSTYPE + 2]; |
---|
202 | m_NClasses = 4; |
---|
203 | |
---|
204 | // initialize attributes |
---|
205 | checkAttributes(true, false, false, false, false, false); |
---|
206 | checkAttributes(false, true, false, false, false, false); |
---|
207 | checkAttributes(false, false, true, false, false, false); |
---|
208 | checkAttributes(false, false, false, true, false, false); |
---|
209 | checkAttributes(false, false, false, false, true, false); |
---|
210 | |
---|
211 | // initialize missing values handling |
---|
212 | for (int i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
213 | // does the scheme support this type of class at all? |
---|
214 | if (!canPredict(i)) |
---|
215 | continue; |
---|
216 | |
---|
217 | // 20% missing |
---|
218 | m_handleMissingPredictors[i] = checkMissingPredictors(i, 20, false); |
---|
219 | if (i <= LAST_CLASSTYPE) |
---|
220 | m_handleMissingClass[i] = checkMissingClass(i, 20, false); |
---|
221 | } |
---|
222 | } |
---|
223 | |
---|
224 | /** Called by JUnit after each test method */ |
---|
225 | protected void tearDown() { |
---|
226 | m_Associator = null; |
---|
227 | m_Tester = null; |
---|
228 | m_OptionTester = null; |
---|
229 | m_GOETester = null; |
---|
230 | |
---|
231 | m_weightedInstancesHandler = false; |
---|
232 | m_NominalPredictors = null; |
---|
233 | m_NumericPredictors = null; |
---|
234 | m_StringPredictors = null; |
---|
235 | m_DatePredictors = null; |
---|
236 | m_RelationalPredictors = null; |
---|
237 | m_handleMissingPredictors = null; |
---|
238 | m_handleMissingClass = null; |
---|
239 | m_RegressionResults = null; |
---|
240 | m_NClasses = 4; |
---|
241 | } |
---|
242 | |
---|
243 | /** |
---|
244 | * Used to create an instance of a specific Associator. |
---|
245 | * |
---|
246 | * @return a suitably configured <code>Associator</code> value |
---|
247 | */ |
---|
248 | public abstract Associator getAssociator(); |
---|
249 | |
---|
250 | /** |
---|
251 | * checks whether at least one attribute type can be handled with the |
---|
252 | * given class type |
---|
253 | * |
---|
254 | * @param type the class type to check for |
---|
255 | * @return true if at least one attribute type can be predicted with |
---|
256 | * the given class |
---|
257 | */ |
---|
258 | protected boolean canPredict(int type) { |
---|
259 | return m_NominalPredictors[type] |
---|
260 | || m_NumericPredictors[type] |
---|
261 | || m_StringPredictors[type] |
---|
262 | || m_DatePredictors[type] |
---|
263 | || m_RelationalPredictors[type]; |
---|
264 | } |
---|
265 | |
---|
266 | /** |
---|
267 | * returns a string for the class type |
---|
268 | * |
---|
269 | * @param type the class type |
---|
270 | * @return the class type as string |
---|
271 | */ |
---|
272 | protected String getClassTypeString(int type) { |
---|
273 | if (type == LAST_CLASSTYPE + 1) |
---|
274 | return "no"; |
---|
275 | else |
---|
276 | return CheckAssociator.attributeTypeToString(type); |
---|
277 | } |
---|
278 | |
---|
279 | /** |
---|
280 | * tests whether the Associator can handle certain attributes and if not, |
---|
281 | * if the exception is OK |
---|
282 | * |
---|
283 | * @param nom to check for nominal attributes |
---|
284 | * @param num to check for numeric attributes |
---|
285 | * @param str to check for string attributes |
---|
286 | * @param dat to check for date attributes |
---|
287 | * @param rel to check for relational attributes |
---|
288 | * @param allowFail whether a junit fail can be executed |
---|
289 | * @see CheckAssociator#canPredict(boolean, boolean, boolean, boolean, boolean, boolean, int) |
---|
290 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
291 | */ |
---|
292 | protected void checkAttributes(boolean nom, boolean num, boolean str, |
---|
293 | boolean dat, boolean rel, |
---|
294 | boolean allowFail) { |
---|
295 | boolean[] result; |
---|
296 | String att; |
---|
297 | int i; |
---|
298 | int type; |
---|
299 | |
---|
300 | // determine text for type of attributes |
---|
301 | att = ""; |
---|
302 | if (nom) |
---|
303 | att = "nominal"; |
---|
304 | else if (num) |
---|
305 | att = "numeric"; |
---|
306 | else if (str) |
---|
307 | att = "string"; |
---|
308 | else if (dat) |
---|
309 | att = "date"; |
---|
310 | else if (rel) |
---|
311 | att = "relational"; |
---|
312 | |
---|
313 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
314 | if (i == LAST_CLASSTYPE + 1) |
---|
315 | type = CheckAssociator.NO_CLASS; |
---|
316 | else |
---|
317 | type = i; |
---|
318 | result = m_Tester.canPredict(nom, num, str, dat, rel, m_multiInstanceHandler, type); |
---|
319 | |
---|
320 | if (nom) |
---|
321 | m_NominalPredictors[i] = result[0]; |
---|
322 | else if (num) |
---|
323 | m_NumericPredictors[i] = result[0]; |
---|
324 | else if (str) |
---|
325 | m_StringPredictors[i] = result[0]; |
---|
326 | else if (dat) |
---|
327 | m_DatePredictors[i] = result[0]; |
---|
328 | else if (rel) |
---|
329 | m_RelationalPredictors[i] = result[0]; |
---|
330 | |
---|
331 | if (!result[0] && !result[1] && allowFail) |
---|
332 | fail("Error handling " + att + " attributes (" + getClassTypeString(i) |
---|
333 | + " class)!"); |
---|
334 | } |
---|
335 | } |
---|
336 | |
---|
337 | /** |
---|
338 | * tests whether the Associator can handle different types of attributes and |
---|
339 | * if not, if the exception is OK |
---|
340 | * |
---|
341 | * @see #checkAttributes(boolean, boolean, boolean, boolean, boolean, boolean) |
---|
342 | */ |
---|
343 | public void testAttributes() { |
---|
344 | // nominal |
---|
345 | checkAttributes(true, false, false, false, false, true); |
---|
346 | // numeric |
---|
347 | checkAttributes(false, true, false, false, false, true); |
---|
348 | // string |
---|
349 | checkAttributes(false, false, true, false, false, true); |
---|
350 | // date |
---|
351 | checkAttributes(false, false, false, true, false, true); |
---|
352 | // relational |
---|
353 | if (!m_multiInstanceHandler) |
---|
354 | checkAttributes(false, false, false, false, true, true); |
---|
355 | } |
---|
356 | |
---|
357 | /** |
---|
358 | * tests whether the scheme declares a serialVersionUID. |
---|
359 | */ |
---|
360 | public void testSerialVersionUID() { |
---|
361 | boolean[] result; |
---|
362 | |
---|
363 | result = m_Tester.declaresSerialVersionUID(); |
---|
364 | |
---|
365 | if (!result[0]) |
---|
366 | fail("Doesn't declare serialVersionUID!"); |
---|
367 | } |
---|
368 | |
---|
369 | /** |
---|
370 | * tests whether the Associator handles instance weights correctly |
---|
371 | * |
---|
372 | * @see CheckAssociator#instanceWeights(boolean, boolean, boolean, boolean, boolean, boolean, int) |
---|
373 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
374 | */ |
---|
375 | public void testInstanceWeights() { |
---|
376 | boolean[] result; |
---|
377 | int i; |
---|
378 | int type; |
---|
379 | |
---|
380 | if (m_weightedInstancesHandler) { |
---|
381 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
382 | // does the Associator support this type of class at all? |
---|
383 | if (!canPredict(i)) |
---|
384 | continue; |
---|
385 | |
---|
386 | if (i == LAST_CLASSTYPE + 1) |
---|
387 | type = CheckAssociator.NO_CLASS; |
---|
388 | else |
---|
389 | type = i; |
---|
390 | |
---|
391 | result = m_Tester.instanceWeights( |
---|
392 | m_NominalPredictors[i], |
---|
393 | m_NumericPredictors[i], |
---|
394 | m_StringPredictors[i], |
---|
395 | m_DatePredictors[i], |
---|
396 | m_RelationalPredictors[i], |
---|
397 | m_multiInstanceHandler, |
---|
398 | type); |
---|
399 | |
---|
400 | if (!result[0]) |
---|
401 | System.err.println("Error handling instance weights (" + getClassTypeString(i) |
---|
402 | + " class)!"); |
---|
403 | } |
---|
404 | } |
---|
405 | } |
---|
406 | |
---|
407 | /** |
---|
408 | * tests whether Associator handles N classes |
---|
409 | * |
---|
410 | * @see CheckAssociator#canHandleNClasses(boolean, boolean, boolean, boolean, boolean, boolean, int) |
---|
411 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
412 | * @see #m_NClasses |
---|
413 | */ |
---|
414 | public void testNClasses() { |
---|
415 | boolean[] result; |
---|
416 | |
---|
417 | if (!canPredict(Attribute.NOMINAL)) |
---|
418 | return; |
---|
419 | |
---|
420 | result = m_Tester.canHandleNClasses( |
---|
421 | m_NominalPredictors[Attribute.NOMINAL], |
---|
422 | m_NumericPredictors[Attribute.NOMINAL], |
---|
423 | m_StringPredictors[Attribute.NOMINAL], |
---|
424 | m_DatePredictors[Attribute.NOMINAL], |
---|
425 | m_RelationalPredictors[Attribute.NOMINAL], |
---|
426 | m_multiInstanceHandler, |
---|
427 | m_NClasses); |
---|
428 | |
---|
429 | if (!result[0] && !result[1]) |
---|
430 | fail("Error handling " + m_NClasses + " classes!"); |
---|
431 | } |
---|
432 | |
---|
433 | /** |
---|
434 | * checks whether the Associator can handle the class attribute at a given |
---|
435 | * position (0-based index, -1 means last). |
---|
436 | * |
---|
437 | * @param type the class type |
---|
438 | * @param position the position of the class attribute (0-based, -1 means last) |
---|
439 | * @return true if the Associator can handle it |
---|
440 | */ |
---|
441 | protected boolean checkClassAsNthAttribute(int type, int position) { |
---|
442 | boolean[] result; |
---|
443 | String indexStr; |
---|
444 | |
---|
445 | result = m_Tester.canHandleClassAsNthAttribute( |
---|
446 | m_NominalPredictors[type], |
---|
447 | m_NumericPredictors[type], |
---|
448 | m_StringPredictors[type], |
---|
449 | m_DatePredictors[type], |
---|
450 | m_RelationalPredictors[type], |
---|
451 | m_multiInstanceHandler, |
---|
452 | type, |
---|
453 | position); |
---|
454 | |
---|
455 | if (position == -1) |
---|
456 | indexStr = "last"; |
---|
457 | else |
---|
458 | indexStr = (position + 1) + "."; |
---|
459 | |
---|
460 | if (!result[0] && !result[1]) |
---|
461 | fail("Error handling class as " + indexStr + " attribute (" |
---|
462 | + getClassTypeString(type) + " class)!"); |
---|
463 | |
---|
464 | return result[0]; |
---|
465 | } |
---|
466 | |
---|
467 | /** |
---|
468 | * Tests whether the Associator can handle class attributes as Nth |
---|
469 | * attribute. In case of multi-instance Associators it performs no tests, |
---|
470 | * since the multi-instance data has a fixed format (bagID,bag,class). |
---|
471 | * |
---|
472 | * @see CheckAssociator#canHandleClassAsNthAttribute(boolean, boolean, boolean, boolean, boolean, boolean, int, int) |
---|
473 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
474 | */ |
---|
475 | public void testClassAsNthAttribute() { |
---|
476 | int i; |
---|
477 | |
---|
478 | // multi-Instance data has fixed format! |
---|
479 | if (m_multiInstanceHandler) |
---|
480 | return; |
---|
481 | |
---|
482 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) { |
---|
483 | // does the Associator support this type of class at all? |
---|
484 | if (!canPredict(i)) |
---|
485 | continue; |
---|
486 | |
---|
487 | // first attribute |
---|
488 | checkClassAsNthAttribute(i, 0); |
---|
489 | |
---|
490 | // second attribute |
---|
491 | checkClassAsNthAttribute(i, 1); |
---|
492 | } |
---|
493 | } |
---|
494 | |
---|
495 | /** |
---|
496 | * tests whether the Associator can handle zero training instances |
---|
497 | * |
---|
498 | * @see CheckAssociator#canHandleZeroTraining(boolean, boolean, boolean, boolean, boolean, boolean, int) |
---|
499 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
500 | */ |
---|
501 | public void testZeroTraining() { |
---|
502 | boolean[] result; |
---|
503 | int i; |
---|
504 | int type; |
---|
505 | |
---|
506 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
507 | // does the Associator support this type of class at all? |
---|
508 | if (!canPredict(i)) |
---|
509 | continue; |
---|
510 | |
---|
511 | if (i == LAST_CLASSTYPE + 1) |
---|
512 | type = CheckAssociator.NO_CLASS; |
---|
513 | else |
---|
514 | type = i; |
---|
515 | |
---|
516 | result = m_Tester.canHandleZeroTraining( |
---|
517 | m_NominalPredictors[i], |
---|
518 | m_NumericPredictors[i], |
---|
519 | m_StringPredictors[i], |
---|
520 | m_DatePredictors[i], |
---|
521 | m_RelationalPredictors[i], |
---|
522 | m_multiInstanceHandler, |
---|
523 | type); |
---|
524 | |
---|
525 | if (!result[0] && !result[1]) |
---|
526 | fail("Error handling zero training instances (" + getClassTypeString(i) |
---|
527 | + " class)!"); |
---|
528 | } |
---|
529 | } |
---|
530 | |
---|
531 | /** |
---|
532 | * checks whether the Associator can handle the given percentage of |
---|
533 | * missing predictors |
---|
534 | * |
---|
535 | * @param type the class type |
---|
536 | * @param percent the percentage of missing predictors |
---|
537 | * @param allowFail if true a fail statement may be executed |
---|
538 | * @return true if the Associator can handle it |
---|
539 | */ |
---|
540 | protected boolean checkMissingPredictors(int type, int percent, boolean allowFail) { |
---|
541 | boolean[] result; |
---|
542 | int classType; |
---|
543 | |
---|
544 | if (type == LAST_CLASSTYPE + 1) |
---|
545 | classType = CheckAssociator.NO_CLASS; |
---|
546 | else |
---|
547 | classType = type; |
---|
548 | |
---|
549 | result = m_Tester.canHandleMissing( |
---|
550 | m_NominalPredictors[type], |
---|
551 | m_NumericPredictors[type], |
---|
552 | m_StringPredictors[type], |
---|
553 | m_DatePredictors[type], |
---|
554 | m_RelationalPredictors[type], |
---|
555 | m_multiInstanceHandler, |
---|
556 | classType, |
---|
557 | true, |
---|
558 | false, |
---|
559 | percent); |
---|
560 | |
---|
561 | if (allowFail) { |
---|
562 | if (!result[0] && !result[1]) |
---|
563 | fail("Error handling " + percent + "% missing predictors (" |
---|
564 | + getClassTypeString(type) + " class)!"); |
---|
565 | } |
---|
566 | |
---|
567 | return result[0]; |
---|
568 | } |
---|
569 | |
---|
570 | /** |
---|
571 | * tests whether the Associator can handle missing predictors (20% and 100%) |
---|
572 | * |
---|
573 | * @see CheckAssociator#canHandleMissing(boolean, boolean, boolean, boolean, boolean, boolean, int, boolean, boolean, int) |
---|
574 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
575 | */ |
---|
576 | public void testMissingPredictors() { |
---|
577 | int i; |
---|
578 | |
---|
579 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
580 | // does the Associator support this type of class at all? |
---|
581 | if (!canPredict(i)) |
---|
582 | continue; |
---|
583 | |
---|
584 | // 20% missing |
---|
585 | checkMissingPredictors(i, 20, true); |
---|
586 | |
---|
587 | // 100% missing |
---|
588 | if (m_handleMissingPredictors[i]) |
---|
589 | checkMissingPredictors(i, 100, true); |
---|
590 | } |
---|
591 | } |
---|
592 | |
---|
593 | /** |
---|
594 | * checks whether the Associator can handle the given percentage of |
---|
595 | * missing class labels |
---|
596 | * |
---|
597 | * @param type the class type |
---|
598 | * @param percent the percentage of missing class labels |
---|
599 | * @param allowFail if true a fail statement may be executed |
---|
600 | * @return true if the Associator can handle it |
---|
601 | */ |
---|
602 | protected boolean checkMissingClass(int type, int percent, boolean allowFail) { |
---|
603 | boolean[] result; |
---|
604 | |
---|
605 | result = m_Tester.canHandleMissing( |
---|
606 | m_NominalPredictors[type], |
---|
607 | m_NumericPredictors[type], |
---|
608 | m_StringPredictors[type], |
---|
609 | m_DatePredictors[type], |
---|
610 | m_RelationalPredictors[type], |
---|
611 | m_multiInstanceHandler, |
---|
612 | type, |
---|
613 | false, |
---|
614 | true, |
---|
615 | percent); |
---|
616 | |
---|
617 | if (allowFail) { |
---|
618 | if (!result[0] && !result[1]) |
---|
619 | fail("Error handling " + percent + "% missing class labels (" |
---|
620 | + getClassTypeString(type) + " class)!"); |
---|
621 | } |
---|
622 | |
---|
623 | return result[0]; |
---|
624 | } |
---|
625 | |
---|
626 | /** |
---|
627 | * tests whether the Associator can handle missing class values (20% and |
---|
628 | * 100%) |
---|
629 | * |
---|
630 | * @see CheckAssociator#canHandleMissing(boolean, boolean, boolean, boolean, boolean, boolean, int, boolean, boolean, int) |
---|
631 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
632 | */ |
---|
633 | public void testMissingClass() { |
---|
634 | int i; |
---|
635 | |
---|
636 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE; i++) { |
---|
637 | // does the Associator support this type of class at all? |
---|
638 | if (!canPredict(i)) |
---|
639 | continue; |
---|
640 | |
---|
641 | // 20% missing |
---|
642 | checkMissingClass(i, 20, true); |
---|
643 | |
---|
644 | // 100% missing |
---|
645 | if (m_handleMissingClass[i]) |
---|
646 | checkMissingClass(i, 100, true); |
---|
647 | } |
---|
648 | } |
---|
649 | |
---|
650 | /** |
---|
651 | * tests whether the Associator correctly initializes in the |
---|
652 | * buildAssociator method |
---|
653 | * |
---|
654 | * @see CheckAssociator#correctBuildInitialisation(boolean, boolean, boolean, boolean, boolean, boolean, int) |
---|
655 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
656 | */ |
---|
657 | public void testBuildInitialization() { |
---|
658 | boolean[] result; |
---|
659 | int i; |
---|
660 | int type; |
---|
661 | |
---|
662 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
663 | // does the Associator support this type of class at all? |
---|
664 | if (!canPredict(i)) |
---|
665 | continue; |
---|
666 | |
---|
667 | if (i == LAST_CLASSTYPE + 1) |
---|
668 | type = CheckAssociator.NO_CLASS; |
---|
669 | else |
---|
670 | type = i; |
---|
671 | |
---|
672 | result = m_Tester.correctBuildInitialisation( |
---|
673 | m_NominalPredictors[i], |
---|
674 | m_NumericPredictors[i], |
---|
675 | m_StringPredictors[i], |
---|
676 | m_DatePredictors[i], |
---|
677 | m_RelationalPredictors[i], |
---|
678 | m_multiInstanceHandler, |
---|
679 | type); |
---|
680 | |
---|
681 | if (!result[0] && !result[1]) |
---|
682 | fail("Incorrect build initialization (" + getClassTypeString(i) |
---|
683 | + " class)!"); |
---|
684 | } |
---|
685 | } |
---|
686 | |
---|
687 | /** |
---|
688 | * tests whether the Associator alters the training set during training. |
---|
689 | * |
---|
690 | * @see CheckAssociator#datasetIntegrity(boolean, boolean, boolean, boolean, boolean, boolean, int, boolean, boolean) |
---|
691 | * @see CheckAssociator#testsPerClassType(int, boolean, boolean) |
---|
692 | */ |
---|
693 | public void testDatasetIntegrity() { |
---|
694 | boolean[] result; |
---|
695 | int i; |
---|
696 | int type; |
---|
697 | |
---|
698 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
699 | // does the Associator support this type of class at all? |
---|
700 | if (!canPredict(i)) |
---|
701 | continue; |
---|
702 | |
---|
703 | if (i == LAST_CLASSTYPE + 1) |
---|
704 | type = CheckAssociator.NO_CLASS; |
---|
705 | else |
---|
706 | type = i; |
---|
707 | |
---|
708 | result = m_Tester.datasetIntegrity( |
---|
709 | m_NominalPredictors[i], |
---|
710 | m_NumericPredictors[i], |
---|
711 | m_StringPredictors[i], |
---|
712 | m_DatePredictors[i], |
---|
713 | m_RelationalPredictors[i], |
---|
714 | m_multiInstanceHandler, |
---|
715 | type, |
---|
716 | m_handleMissingPredictors[i], |
---|
717 | m_handleMissingClass[i]); |
---|
718 | |
---|
719 | if (!result[0] && !result[1]) |
---|
720 | fail("Training set is altered during training (" |
---|
721 | + getClassTypeString(i) + " class)!"); |
---|
722 | } |
---|
723 | } |
---|
724 | |
---|
725 | /** |
---|
726 | * Builds a model using the current Associator using the given data and |
---|
727 | * returns the produced output. |
---|
728 | * TODO: unified rules as output instead of toString() result??? |
---|
729 | * |
---|
730 | * @param data the instances to test the Associator on |
---|
731 | * @return a String containing the output of the Associator. |
---|
732 | * @throws Exception if something goes wrong |
---|
733 | */ |
---|
734 | protected String useAssociator(Instances data) throws Exception { |
---|
735 | Associator associator = null; |
---|
736 | |
---|
737 | try { |
---|
738 | associator = AbstractAssociator.makeCopy(m_Associator); |
---|
739 | } |
---|
740 | catch (Exception e) { |
---|
741 | e.printStackTrace(); |
---|
742 | fail("Problem setting up to use Associator: " + e); |
---|
743 | } |
---|
744 | |
---|
745 | associator.buildAssociations(data); |
---|
746 | |
---|
747 | return associator.toString(); |
---|
748 | } |
---|
749 | |
---|
750 | /** |
---|
751 | * Provides a hook for derived classes to further modify the data. Currently, |
---|
752 | * the data is just passed through. |
---|
753 | * |
---|
754 | * @param data the data to process |
---|
755 | * @return the processed data |
---|
756 | */ |
---|
757 | protected Instances process(Instances data) { |
---|
758 | return data; |
---|
759 | } |
---|
760 | |
---|
761 | /** |
---|
762 | * Runs a regression test -- this checks that the output of the tested |
---|
763 | * object matches that in a reference version. When this test is |
---|
764 | * run without any pre-existing reference output, the reference version |
---|
765 | * is created. |
---|
766 | * |
---|
767 | * @throws Exception if something goes wrong |
---|
768 | */ |
---|
769 | public void testRegression() throws Exception { |
---|
770 | int i; |
---|
771 | boolean succeeded; |
---|
772 | Regression reg; |
---|
773 | Instances train; |
---|
774 | int type; |
---|
775 | |
---|
776 | // don't bother if not working correctly |
---|
777 | if (m_Tester.hasClasspathProblems()) |
---|
778 | return; |
---|
779 | |
---|
780 | reg = new Regression(this.getClass()); |
---|
781 | succeeded = false; |
---|
782 | train = null; |
---|
783 | |
---|
784 | for (i = FIRST_CLASSTYPE; i <= LAST_CLASSTYPE + 1; i++) { |
---|
785 | // does the Associator support this type of class at all? |
---|
786 | if (!canPredict(i)) |
---|
787 | continue; |
---|
788 | |
---|
789 | if (i == LAST_CLASSTYPE + 1) |
---|
790 | type = CheckAssociator.NO_CLASS; |
---|
791 | else |
---|
792 | type = i; |
---|
793 | |
---|
794 | train = m_Tester.makeTestDataset( |
---|
795 | 42, m_Tester.getNumInstances(), |
---|
796 | m_NominalPredictors[i] ? 2 : 0, |
---|
797 | m_NumericPredictors[i] ? 1 : 0, |
---|
798 | m_StringPredictors[i] ? 1 : 0, |
---|
799 | m_DatePredictors[i] ? 1 : 0, |
---|
800 | m_RelationalPredictors[i] ? 1 : 0, |
---|
801 | 2, |
---|
802 | type, |
---|
803 | m_multiInstanceHandler); |
---|
804 | |
---|
805 | try { |
---|
806 | m_RegressionResults[i] = useAssociator(train); |
---|
807 | succeeded = true; |
---|
808 | reg.println(m_RegressionResults[i]); |
---|
809 | } |
---|
810 | catch (Exception e) { |
---|
811 | String msg = e.getMessage().toLowerCase(); |
---|
812 | if (msg.indexOf("not in classpath") > -1) |
---|
813 | return; |
---|
814 | |
---|
815 | m_RegressionResults[i] = null; |
---|
816 | } |
---|
817 | } |
---|
818 | |
---|
819 | if (!succeeded) { |
---|
820 | fail("Problem during regression testing: no successful predictions for any class type"); |
---|
821 | } |
---|
822 | |
---|
823 | try { |
---|
824 | String diff = reg.diff(); |
---|
825 | if (diff == null) { |
---|
826 | System.err.println("Warning: No reference available, creating."); |
---|
827 | } else if (!diff.equals("")) { |
---|
828 | fail("Regression test failed. Difference:\n" + diff); |
---|
829 | } |
---|
830 | } |
---|
831 | catch (java.io.IOException ex) { |
---|
832 | fail("Problem during regression testing.\n" + ex); |
---|
833 | } |
---|
834 | } |
---|
835 | |
---|
836 | /** |
---|
837 | * tests the listing of the options |
---|
838 | */ |
---|
839 | public void testListOptions() { |
---|
840 | if (m_OptionTester.getOptionHandler() != null) { |
---|
841 | if (!m_OptionTester.checkListOptions()) |
---|
842 | fail("Options cannot be listed via listOptions."); |
---|
843 | } |
---|
844 | } |
---|
845 | |
---|
846 | /** |
---|
847 | * tests the setting of the options |
---|
848 | */ |
---|
849 | public void testSetOptions() { |
---|
850 | if (m_OptionTester.getOptionHandler() != null) { |
---|
851 | if (!m_OptionTester.checkSetOptions()) |
---|
852 | fail("setOptions method failed."); |
---|
853 | } |
---|
854 | } |
---|
855 | |
---|
856 | /** |
---|
857 | * tests whether the default settings are processed correctly |
---|
858 | */ |
---|
859 | public void testDefaultOptions() { |
---|
860 | if (m_OptionTester.getOptionHandler() != null) { |
---|
861 | if (!m_OptionTester.checkDefaultOptions()) |
---|
862 | fail("Default options were not processed correctly."); |
---|
863 | } |
---|
864 | } |
---|
865 | |
---|
866 | /** |
---|
867 | * tests whether there are any remaining options |
---|
868 | */ |
---|
869 | public void testRemainingOptions() { |
---|
870 | if (m_OptionTester.getOptionHandler() != null) { |
---|
871 | if (!m_OptionTester.checkRemainingOptions()) |
---|
872 | fail("There were 'left-over' options."); |
---|
873 | } |
---|
874 | } |
---|
875 | |
---|
876 | /** |
---|
877 | * tests the whether the user-supplied options stay the same after setting. |
---|
878 | * getting, and re-setting again. |
---|
879 | * |
---|
880 | * @see #getOptionTester() |
---|
881 | */ |
---|
882 | public void testCanonicalUserOptions() { |
---|
883 | if (m_OptionTester.getOptionHandler() != null) { |
---|
884 | if (!m_OptionTester.checkCanonicalUserOptions()) |
---|
885 | fail("setOptions method failed"); |
---|
886 | } |
---|
887 | } |
---|
888 | |
---|
889 | /** |
---|
890 | * tests the resetting of the options to the default ones |
---|
891 | */ |
---|
892 | public void testResettingOptions() { |
---|
893 | if (m_OptionTester.getOptionHandler() != null) { |
---|
894 | if (!m_OptionTester.checkSetOptions()) |
---|
895 | fail("Resetting of options failed"); |
---|
896 | } |
---|
897 | } |
---|
898 | |
---|
899 | /** |
---|
900 | * tests for a globalInfo method |
---|
901 | */ |
---|
902 | public void testGlobalInfo() { |
---|
903 | if (!m_GOETester.checkGlobalInfo()) |
---|
904 | fail("No globalInfo method"); |
---|
905 | } |
---|
906 | |
---|
907 | /** |
---|
908 | * tests the tool tips |
---|
909 | */ |
---|
910 | public void testToolTips() { |
---|
911 | if (!m_GOETester.checkToolTips()) |
---|
912 | fail("Tool tips inconsistent"); |
---|
913 | } |
---|
914 | } |
---|