001/* Run the Ptolemy model tests in the auto/ directory using JUnit. 002 003 Copyright (c) 2011-2018 The Regents of the University of California. 004 All rights reserved. 005 Permission is hereby granted, without written agreement and without 006 license or royalty fees, to use, copy, modify, and distribute this 007 software and its documentation for any purpose, provided that the above 008 copyright notice and the following two paragraphs appear in all copies 009 of this software. 010 011 IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY 012 FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES 013 ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF 014 THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF 015 SUCH DAMAGE. 016 017 THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 018 INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 019 MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE 020 PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF 021 CALIFORNIA HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, 022 ENHANCEMENTS, OR MODIFICATIONS. 023 024 PT_COPYRIGHT_VERSION_2 025 COPYRIGHTENDKEY 026 027*/ 028 029package ptolemy.util.test.junit; 030 031import java.lang.reflect.Method; 032 033import org.junit.AfterClass; 034import org.junit.Test; 035import org.junit.runner.RunWith; 036 037import junitparams.JUnitParamsRunner; 038import junitparams.Parameters; 039import ptolemy.util.StringUtilities; 040 041/////////////////////////////////////////////////////////////////// 042//// AutoTests 043/** 044 * Run the Ptolemy model tests in the auto/ directory using JUnit. 045 * 046 * <p> 047 * This test must be run from the directory that contains the auto/ directory, 048 * for example: 049 * </p> 050 * 051 * <pre> 052 * (cd ~/ptII/ptolemy/actor/lib/io/test; java -classpath ${PTII}:${PTII}/lib/junit-4.8.2.jar:${PTII}/lib/JUnitParams-0.3.0.jar org.junit.runner.JUnitCore ptolemy.util.test.junit.AutoTests) 053 * </pre> 054 * 055 * <p> 056 * This test uses JUnitParams from <a 057 * href="http://code.google.com/p/junitparams/#in_browser" 058 * >http://code.google.com/p/junitparams/</a>, which is released under <a 059 * href="http://www.apache.org/licenses/LICENSE-2.0#in_browser">Apache License 060 * 2.0</a>. 061 * </p> 062 * 063 * @author Christopher Brooks 064 * @version $Id$ 065 * @since Ptolemy II 10.0 066 * @Pt.ProposedRating Red (cxh) 067 * @Pt.AcceptedRating Red (cxh) 068 */ 069@RunWith(JUnitParamsRunner.class) 070public class AutoTests extends ModelTests { 071 072 /** If the VertxHelperBase class is present, then invoke the 073 * closeVertx() method so that this process does not wait around 074 * for the Vert.x threads. 075 */ 076 @AfterClass 077 public static void afterClass() { 078 try { 079 Class clazz = Class 080 .forName("ptolemy.actor.lib.jjs.VertxHelperBase"); 081 if (clazz != null) { 082 Method method = clazz.getMethod("closeVertx"); 083 System.out.println("AutoTests.java: About to close Vertx."); 084 method.invoke(null); 085 System.out.println("AutoTests.java: Vertx closed."); 086 } 087 } catch (NoClassDefFoundError ex) { 088 // Ignore this, it means that MoMLSimpleApplication was invoked without the Vert.x jar files. 089 } catch (Throwable throwable) { 090 System.err.println( 091 "AutoTests: Failed to invoke VertxHelperBase.closeVertx() during exit. This can be ignored. Error was: " 092 + throwable); 093 } 094 095 } 096 097 /** If the fullPath is a hlacerti or accessor demo, then 098 * delay so that the system can stabilize. 099 * @param fullPath The forward slash separated path of the demo. 100 */ 101 public static void delayIfNecessary(String fullPath) { 102 int delay = 5000; 103 boolean match = fullPath.matches( 104 ".*(org/hlacerti|org/terraswarm/accessors|ptolemy/actor/lib/jjs).*"); 105 // System.out.println("AutoTests.java: fullPath: " + fullPath 106 // + " match: " + match); 107 if (match) { 108 System.out.println("----------------- " + (new java.util.Date()) 109 + " About to sleep for " + delay / 1000.0 110 + " seconds before running or rerunning. Test is: " 111 + fullPath); 112 try { 113 Thread.sleep(delay); 114 } catch (InterruptedException ex) { 115 System.err.println( 116 "Sleep before rerunning was interrupted: " + ex); 117 } 118 System.out.println("Done sleeping"); 119 } 120 } 121 122 /** 123 * Execute a model and time out after 900000 ms. 124 * 125 * For information about how this class is used in the Travis build, 126 * see <a href="https://wiki.eecs.berkeley.edu/ptexternal/Main/Travis#in_browser target="_top">https://wiki.eecs.berkeley.edu/ptexternal/Main/Travis</a>. 127 * 128 * @param fullPath 129 * The full path to the model file to be executed. If the 130 * fullPath ends with the value of the 131 * {@link #THERE_ARE_NO_AUTO_TESTS}, then the method returns 132 * immediately. 133 * @exception Throwable 134 * If thrown while executing the model. 135 */ 136 @Test(timeout = 900000) 137 @Parameters(method = "modelValues") 138 public void RunModel(String fullPath) throws Throwable { 139 if (fullPath.endsWith(THERE_ARE_NO_AUTO_TESTS)) { 140 System.out.println( 141 "No auto/*.xml tests in " + System.getProperty("user.dir")); 142 return; 143 } 144 if (fullPath.endsWith(THERE_ARE_NO_AUTO_ARCH_TESTS)) { 145 System.out.println("No " + AutoNameArchTests.autoNameArch() 146 + "*.xml tests in " + System.getProperty("user.dir")); 147 return; 148 } 149 150 // Only check for the JSAccessor class if there are auto tests. 151 // MoMLParser.tcl had a test that was reporting different 152 // results between running using ptjacl and JUnit because 153 // JUnit was always instantiating a JSAccessor, even 154 // if there were no auto tests. This was causing 155 // "StreamChangeRequest.changeExecuted(): AccessorIconLoader succeeded" 156 // to appear in the listener. 157 if (!AutoTests._haveCheckedForJSAccessor) { 158 AutoTests._haveCheckedForJSAccessor = true; 159 AutoTests._checkForJSAccessor(); 160 } 161 // Use a whole row of equals to signify the start of a new test. 162 System.out.println( 163 "\n==========================================================================================="); 164 if (modelFileIsOK(fullPath)) { 165 166 System.out.println("----------------- testing " 167 + (new java.util.Date()) + " " + fullPath); 168 System.out.flush(); 169 if (_applicationConstructor == null) { 170 // Delay instantiating MoMLSimpleApplication so that we 171 // can run the kernel tests without requiring moml 172 _applicationClass = Class 173 .forName("ptolemy.moml.MoMLSimpleApplication"); 174 _applicationConstructor = _applicationClass 175 .getConstructor(String.class); 176 } 177 178 // _applicationConstructor might have been initialized in 179 // AutoKnownFailedTests, so we initialize 180 // _applicationToplevelMethod here. 181 if (_applicationToplevelMethod == null) { 182 _applicationToplevelMethod = _applicationClass 183 .getMethod("toplevel", new Class[] {}); 184 } 185 186 // If a model is in various directories, including 187 // org/terraswarm/accessors/test, the delay before 188 // reloading. See 189 // https://wiki.eecs.berkeley.edu/ptexternal/Main/Main/WebSocketDeadlock#Starvation 190 AutoTests.delayIfNecessary(fullPath); 191 192 System.out.println("----------------- Instantiating " + fullPath); 193 Object instance = _applicationConstructor.newInstance(fullPath); 194 Method rerunMethod = _applicationClass.getMethod("rerun", 195 (Class<?>[]) null); 196 197 // If JSAccessor is present and the model contains one, then 198 // reload all the JSAccessors and rerun the model 199 200 if (_jsAccessorClass == null) { 201 // JsAccessor is not present, just rerun. 202 System.out 203 .println("----------------- testing again " + fullPath); 204 System.out.flush(); 205 rerunMethod.invoke(instance, (Object[]) null); 206 207 } else { 208 // JSAccessor is present, reload and rerun. 209 210 System.out.println( 211 "----------------- Invoking toplevel() on " + instance); 212 _applicationToplevelMethod = _applicationClass 213 .getMethod("toplevel", new Class[] {}); 214 Object toplevel = _applicationToplevelMethod.invoke(instance, 215 (Object[]) null); 216 System.out.println( 217 "----------------- Done invoking toplevel() on " 218 + instance); 219 220 if (_jsAccessorReloadAllAccessorsMethod == null) { 221 throw new InternalError( 222 "Found the JSAccessor class, but not the reloadAllAccessors() method?"); 223 } 224 // Reload all the accessors and invoke rerun. 225 if (((Boolean) _jsAccessorReloadAllAccessorsMethod.invoke(null, 226 new Object[] { toplevel })).booleanValue()) { 227 System.out.println( 228 "-------------- Reloaded accessors, but skipping rerun for now. " 229 + (new java.util.Date()) + " " + fullPath); 230 System.out.flush(); 231 // Autotests.delayIfNecessary(fullPath); 232 233 // System.out.println( 234 // "----------------- Reloaded Accessors and testing again " + (new java.util.Date()) + " " 235 // + fullPath); 236 // System.out.flush(); 237 // rerunMethod.invoke(instance, (Object[]) null); 238 } 239 } 240 } 241 } 242 243 /** Return true if the model should be run. 244 * This is a hack to avoid a problem where certain models 245 * interact badly with the Cobertura code coverage tool 246 * or with Travis. 247 * @param fullPath The full path of the model to be executed 248 * @return true if the model should be run. 249 */ 250 public boolean modelFileIsOK(String fullPath) { 251 if (fullPath.endsWith("de/test/auto/ThreadedComposite.xml") 252 && !StringUtilities 253 .getProperty("net.sourceforge.cobertura.datafile") 254 .equals("")) { 255 System.out.println("----------------- *** Skipping testing of " 256 + fullPath + " because it interacts badly with Cobertura."); 257 System.out.flush(); 258 return false; 259 } 260 261 // Under Travis, skip certain demos. To see what environment 262 // variables are set by Travis, see 263 // https://docs.travis-ci.com/user/environment-variables/ 264 String travis = System.getenv("TRAVIS"); 265 if (travis != null && travis.equals("true")) { 266 String[] travisSkip = { "org/hlacerti/test/auto", 267 "org/terraswarm/accessor/test/auto/GetDate.xml", 268 "org/terraswarm/accessor/accessors/web/services/test/auto/ReverseGeoCoderTest.xml", 269 "ptolemy/actor/lib/jjs/modules/audio/test/auto/SpeechSynthesisTest.xml", 270 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTComposite.xml", 271 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTGet.xml", 272 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTGetCompleteResponseOnly.xml", 273 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTPost.xml", 274 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTPostDataTypes.xml", 275 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTPut.xml", 276 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTReceiveImage.xml", 277 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTSendImage.xml", 278 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTStringURL.xml", 279 "ptolemy/actor/lib/jjs/modules/httpClient/test/auto/RESTTimeout.xml", 280 "ptolemy/actor/lib/jjs/modules/httpServer/test/auto/KeyValueStoreClient.xml", 281 "ptolemy/actor/lib/jjs/modules/httpServer/test/auto/WebServerBasic.xml", 282 "ptolemy/actor/lib/jjs/modules/httpServer/test/auto/WebServerTimeout.xml", 283 "ptolemy/actor/lib/jjs/modules/socket/test/auto/Message2.xml", 284 "ptolemy/actor/lib/jjs/modules/socket/test/auto/Message3.xml", 285 "ptolemy/actor/lib/jjs/modules/socket/test/auto/Message4.xml", 286 "ptolemy/actor/lib/jjs/modules/socket/test/auto/Socket1.xml", 287 "ptolemy/actor/lib/jjs/modules/socket/test/auto/Socket2.xml", 288 "ptolemy/actor/lib/jjs/modules/socket/test/auto/Socket3.xml", 289 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketByte.xml", 290 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketDouble.xml", 291 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketDoubleArray.xml", 292 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketDoubleArrayBatched.xml", 293 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketFloat.xml", 294 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketImage.xml", 295 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketInt.xml", 296 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketInt.xml", 297 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketImagePNG.xml", 298 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketImageRaw.xml", 299 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketShort.xml", 300 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketStringArray.xml", 301 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketTypicalUsage.xml", 302 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketUnsignedByte.xml", 303 "ptolemy/actor/lib/jjs/modules/socket/test/auto/SocketUnsignedShort.xml", 304 "ptolemy/actor/lib/jjs/modules/socket/test/auto/TCPSocketDoubleArrayBatched.xml", 305 "ptolemy/actor/lib/jjs/modules/socket/test/auto/TCPSocketSecureServerClient.xml", 306 "ptolemy/actor/lib/jjs/modules/socket/test/auto/TCPSocketTwoClients.xml", 307 "ptolemy/actor/lib/jjs/modules/socket/test/auto/TCPSocketUnsignedShort.xml", 308 "ptolemy/actor/lib/jjs/modules/socket/test/auto/TCPSocketUnsignedShortSimple.xml", 309 "ptolemy/actor/lib/jjs/modules/udpSocket/test/auto/UDPSocketInt.xml", 310 "ptolemy/actor/lib/jjs/modules/udpSocket/test/auto/UDPSocketSelf.xml", 311 "ptolemy/actor/lib/jjs/modules/udpSocket/test/auto/UDPSocketString.xml", 312 "ptolemy/actor/lib/jjs/modules/webSocket/test/auto/FullDuplex2.xml", 313 "ptolemy/actor/lib/jjs/modules/webSocket/test/auto/WebSocketClient2JS.xml", 314 "ptolemy/actor/lib/jjs/modules/webSocket/test/auto/WebSocketClient3JS.xml", 315 "ptolemy/actor/lib/jjs/modules/webSocket/test/auto/WebSocketClientJS.xml", 316 "ptolemy/actor/lib/jjs/modules/webSocket/test/auto/WebSocketsServerClient.xml" }; 317 for (String element : travisSkip) { 318 if (fullPath.indexOf(element) != -1) { 319 System.out.println( 320 "----------------- *** Skipping testing of " 321 + fullPath 322 + " because it does fails under Travis. " 323 + "To updated this list, edit ptolemy/util/test/junit/AutoTests.java"); 324 System.out.flush(); 325 return false; 326 } 327 } 328 } 329 return true; 330 } 331 332 /////////////////////////////////////////////////////////////////// 333 //// protected fields //// 334 335 /** The org.terraswarm.accessor.JSAccessor class, which is tested 336 * by reloading Accessors. 337 */ 338 protected static Class<?> _jsAccessorClass = null; 339 340 /** The method that reloads all the accessors in a CompositeEntity. */ 341 protected static Method _jsAccessorReloadAllAccessorsMethod = null; 342 343 /////////////////////////////////////////////////////////////////// 344 //// private methods //// 345 346 private static void _checkForJSAccessor() { 347 try { 348 _jsAccessorClass = Class 349 .forName("org.terraswarm.accessor.JSAccessor"); 350 Class compositeEntityClass = Class 351 .forName("ptolemy.kernel.CompositeEntity"); 352 _jsAccessorReloadAllAccessorsMethod = _jsAccessorClass.getMethod( 353 "reloadAllAccessors", new Class[] { compositeEntityClass }); 354 } catch (Throwable throwable) { 355 // Ignore this, it could be that the JSAccessor class 356 // is not present. 357 _jsAccessorClass = null; 358 _jsAccessorReloadAllAccessorsMethod = null; 359 } 360 } 361 362 /////////////////////////////////////////////////////////////////// 363 //// private fields //// 364 365 /** True if we have checked for JSAccessor. */ 366 private static boolean _haveCheckedForJSAccessor = false; 367}