001/* 002 * Copyright (c) 2009-2012 The Regents of the University of California. 003 * All rights reserved. 004 * 005 * '$Author: crawl $' 006 * '$Date: 2012-11-26 22:19:36 +0000 (Mon, 26 Nov 2012) $' 007 * '$Revision: 31113 $' 008 * 009 * Permission is hereby granted, without written agreement and without 010 * license or royalty fees, to use, copy, modify, and distribute this 011 * software and its documentation for any purpose, provided that the above 012 * copyright notice and the following two paragraphs appear in all copies 013 * of this software. 014 * 015 * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY 016 * FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES 017 * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF 018 * THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF 019 * SUCH DAMAGE. 020 * 021 * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 022 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 023 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE 024 * PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF 025 * CALIFORNIA HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, 026 * ENHANCEMENTS, OR MODIFICATIONS. 027 * 028 */ 029 030package org.sdm.spa.actors.transport; 031 032import java.io.ByteArrayOutputStream; 033import java.io.File; 034import java.io.OutputStream; 035 036import org.apache.commons.logging.Log; 037import org.apache.commons.logging.LogFactory; 038import org.kepler.ssh.ExecException; 039import org.kepler.ssh.SshException; 040import org.kepler.ssh.SshExec; 041import org.sdm.spa.actors.transport.vo.ConnectionDetails; 042 043/** 044 * This class provides methods to copy files across two remote machines using 045 * bbcp. It generates the command based on the user input, connects to the 046 * source machine using ssh and executes the commands. Internally uses 047 * <code>SshExec</code> methods to execute command. 048 * <p> 049 * Copy operation would fail if file with the same name already exists. Command 050 * line options can be set to override the default behavior 051 * <p> 052 * The path of bbcp executable on the remote machine should be set in the class 053 * variable 'protocolPathDest', if bbcp is installed on a non standard path. On 054 * the local machine or remote source machine, if the variable protocolPathSrc 055 * is not set, attempt is made to search the most common locations user's home 056 * directory, /usr/bin, /bin, and /usr/local/bin. If the executable is not found 057 * in the above paths, an error is reported 058 * <p> 059 * 060 * @author Chandrika Sivaramakrishnan 061 * 062 */ 063public class BbcpCopier extends FileCopierBase { 064 065 // /////////////////////Private Variables///////////////////////////////// 066 private static final Log log = LogFactory 067 .getLog(BbcpCopier.class.getName()); 068 private static final boolean isDebugging = log.isDebugEnabled(); 069 070 // //////////////////Protected Methods//////////////////////////////////// 071 072 /* 073 * Generates the bbcp command to copy from remote host to local host. 074 * Executes the command using SshExec class 075 * 076 * @see 077 * org.sdm.spa.actors.transport.FileCopier#copyFrom(org.sdm.spa.actors.transport 078 * .vo.ConnectionDetails, java.lang.String, java.lang.String, boolean) 079 */ 080 @Override 081 // ANUSUA Change -Starts 082 protected CopyResult copyFrom(ConnectionDetails srcDetails, String srcFile, 083 String destFile, boolean recursive) throws SshException { 084 // force copy - overwrite files if already exist 085 cmdLineOptions += "-f"; 086 String osname = (System.getProperty("os.name")).toLowerCase(); 087 /* 088 * if (osname.contains("windows")) { throw new 089 * SshException("BBCP is not supported on Windows machines"); } 090 */ 091 092 int exitCode = 0; 093 StringBuffer cmd = new StringBuffer(100); 094 OutputStream streamOut = new ByteArrayOutputStream(); 095 OutputStream streamErr = new ByteArrayOutputStream(); 096 // Connecting to local destination 097 SshExec localSshObj = new SshExec(System.getProperty("user.name"), 098 "localhost"); 099 localSshObj.setTimeout(timeout, false, false); 100 localSshObj.setPseudoTerminal(true); 101 localSshObj.setForcedCleanUp(forcedCleanup); 102 String cmdWithPath=""; 103 File single_srcFileObj = null; 104 File srcFile_Obj = null; 105 File srcFileObjlst = null; 106 boolean flag = false; 107 String[] srcFileList = null; 108 String[] srcFile_list = null; 109 String wildcardFileList; 110 StringBuffer warn = new StringBuffer(100); 111 boolean warn_flag = false; 112 113 SshExec sshObject = new SshExec(srcDetails.getUser(), srcDetails 114 .getHost(), srcDetails.getPort()); 115 116 if (!(srcFile.contains(","))) { 117 if (srcFile.startsWith("/")) { 118 srcFile_list = new String[1]; 119 srcFile_list[0] = srcFile.trim(); 120 } else { 121 return new CopyResult( 122 1, 123 "", 124 srcFile 125 + " does not contain full path to the file. Please provide full path. "); 126 } 127 } else { 128 if (srcFile.contains(",")) { 129 srcFileList = srcFile.split(","); 130 srcFile_list = new String[srcFileList.length]; 131 for (int count = 0; count < srcFileList.length; count++) { 132 if ((srcFileList[count].trim()).startsWith("/")) { 133 srcFile_list[count] = srcFileList[count].trim(); 134 } else { 135 warn.append(srcFile_list[count].trim() + " "); 136 if (!warn_flag) 137 warn_flag = true; 138 } 139 } 140 } 141 } 142 143 if (warn_flag) { 144 warn 145 .append(" does not contain full path to the source file. Please provide full path. "); 146 warn_flag = false; 147 } 148 149 try { 150 cmd.append("bbcp "); 151 cmd.append(cmdLineOptions); // Review - Check if cmdLineOptions 152 // already has a -z******************#DONE 153 if (!(cmdLineOptions.equals("-z"))) {// dealing with firewall 154 cmd.append(" -z "); 155 } 156 // BBCP does not handle error when directory is specified without 157 // recursive flag. 158 // Better to have recursive always on, rather than giving out wrong 159 // output. 160 cmd.append("-r "); 161 162 if (!protocolPathSrc.equals("")) { 163 cmd.append("-S \"ssh -l %U %H "); 164 cmd.append(protocolPathSrc); 165 cmd.append("bbcp\" "); 166 } 167 if (!protocolPathDest.equals("")) { 168 cmd.append("-T \"ssh -l %U %H "); 169 cmd.append(protocolPathDest); 170 cmd.append("bbcp\" "); 171 } 172 cmd.append(srcDetails.toString()); 173 cmd.append(":"); 174 175 for (int i = 0; i < srcFile_list.length; i++) { 176 if (srcFile_list[i].contains("*") 177 || (srcFile_list[i].contains("+"))) { 178 // BBCP cannot handle wildcard pattern * if copy is 179 // being done from destination to source. We need 180 // function to list files matching the pattern 181 System.out.println("wildcard found in filename :" 182 + srcFile_list[i]); 183 wildcardFileList = sshObject.getwildcardFileListingBBCP( 184 srcDetails.toString(), srcFile_list[i]); 185 cmd.append(wildcardFileList); 186 } else { 187 // quotes if file contains wildcard - it might contain space 188 // in file name 189 cmd.append(srcDetails.toString()); 190 cmd.append(":"); 191 cmd.append("\""); 192 cmd.append(srcFile_list[i]); 193 cmd.append("\""); 194 cmd.append(" "); 195 } 196 } 197 cmd.append(" "); 198 cmd.append(destFile);// Review - check if dest is 199 // dir**********#DONE, Checked earlier 200 201 if (protocolPathDest.equals("")) { 202 cmdWithPath = getCmdWithDefaultPath(cmd); 203 } else { 204 cmdWithPath = protocolPathDest + cmd; 205 } 206 // Execute bbcp command 207 if (isDebugging) 208 log.debug("copy cmd=" + cmdWithPath); 209 210 System.out.println("BBCP command is : " + cmdWithPath); 211 212 exitCode = localSshObj.executeCmd(cmdWithPath, streamOut, 213 streamErr, srcDetails.toString()); 214 215 } catch (Exception e) { 216 return new CopyResult(1, e.toString(), ""); 217 } 218 if (exitCode > 0) { 219 log.error("Output on stdout:" + streamOut); 220 log.error("Output on stderr:" + streamErr); 221 } 222 223 String message = streamErr.toString(); 224 if (message == null || message.trim().equals("")) { 225 message = streamOut.toString(); 226 } 227 return new CopyResult(exitCode, message, warn.toString()); 228 229 } 230 231 /* 232 * Generates the bbcp command to copy from local host to remote host. 233 * Executes the command using SshExec class 234 * 235 * @see org.sdm.spa.actors.transport.FileCopier#copyTo(java.lang.String, 236 * org.sdm.spa.actors.transport.vo.ConnectionDetails, java.lang.String, 237 * boolean) 238 */ 239 @Override 240 protected CopyResult copyTo(String srcFile, ConnectionDetails destDetails, 241 String destFile, boolean recursive) throws SshException { 242 cmdLineOptions = ""; // Review - don't forcefully rewrite. 243 // If user wants to overwrite he can set it as command line option, but 244 // if you 245 // set it, he can't override it with any command line 246 // option*********#DONE 247 String osname = (System.getProperty("os.name")).toLowerCase(); 248 String userhome = (System.getProperty("user.home")).toLowerCase(); 249 if (osname.contains("windows")) { 250 throw new SshException("BBCP is not supported on Windows machines"); 251 } 252 253 int exitCode = 0; 254 String cmdWithPath; 255 File srcFile_Obj = null; 256 boolean flag = false; 257 String[] srcFileList = null; 258 String[] srcFile_list = null; 259 StringBuffer warn = new StringBuffer(100); 260 261 OutputStream streamOut = new ByteArrayOutputStream(); 262 OutputStream streamErr = new ByteArrayOutputStream(); 263 StringBuffer cmd = new StringBuffer(100); 264 SshExec localSshObj = new SshExec(System.getProperty("user.name"), 265 "localhost"); 266 localSshObj.setTimeout(timeout, false, false); 267 localSshObj.setPseudoTerminal(true); 268 localSshObj.setForcedCleanUp(forcedCleanup); 269 System.out.println("Initial str : " + srcFile); 270 if (srcFile.contains(",")) { 271 // Anand: list of files 272 System.out.println("list of files************"); 273 srcFileList = srcFile.split(","); 274 srcFile_list = new String[srcFileList.length]; 275 for (int count = 0; count < srcFileList.length; count++) { 276 System.out.println("before : " + srcFileList[count]); 277 278 if ((srcFileList[count].trim()).startsWith("/")) { 279 srcFile_list[count] = srcFileList[count].trim(); 280 } else { 281 srcFile_list[count] = userhome + "/" 282 + srcFileList[count].trim(); 283 } 284 srcFile_Obj = new File(srcFileList[count]); 285 if (srcFile_Obj.isDirectory()) { 286 flag = true; 287 } 288 System.out.println("after : " + srcFile_list[count]); 289 } 290 } else { 291 System.out.println("single files************"); 292 // Anand: single file 293 srcFile_list = new String[1]; 294 srcFile = srcFile.trim(); 295 if (srcFile.startsWith("/")) { 296 srcFile_list[0] = srcFile; 297 } else { 298 srcFile_list[0] = userhome + "/" + srcFile; 299 } 300 srcFile_Obj = new File(srcFile); 301 if (srcFile_Obj.isDirectory()) { 302 flag = true; 303 } 304 } 305 306 // build bbcp command 307 cmd.append("bbcp "); 308 cmd.append(cmdLineOptions); 309 cmd.append(" "); 310 if (recursive || flag) { 311 cmd.append("-r "); 312 } 313 if (!protocolPathSrc.equals("")) { 314 cmd.append("-S \"ssh -l %U %H "); 315 cmd.append(protocolPathSrc); 316 cmd.append("bbcp\" "); 317 } 318 if (!protocolPathDest.equals("")) { 319 cmd.append("-T \"ssh -l %U %H "); 320 cmd.append(protocolPathDest); 321 cmd.append("bbcp\" "); 322 } 323 // all files are in the srcFile_list (single file too) 324 for (int i = 0; i < srcFile_list.length; i++) { 325 if (srcFile_list[i].contains("*") 326 || (srcFile_list[i].contains("*"))) { 327 // no quotes if file contains wildcard 328 cmd.append(srcFile_list[i]); 329 cmd.append(" "); 330 } else { 331 // quotes if file does not contain wildcard - it might contain 332 // space in file name 333 cmd.append("\""); 334 cmd.append(srcFile_list[i]); 335 cmd.append("\""); 336 cmd.append(" "); 337 } 338 } 339 cmd.append(destDetails.toString()); 340 cmd.append(":"); 341 cmd.append(destFile); 342 343 if (protocolPathSrc.equals("")) { 344 cmdWithPath = getCmdWithDefaultPath(cmd); 345 } else { 346 cmdWithPath = protocolPathSrc + cmd; 347 } 348 System.out.println("BBCP Command to be executed is : " + cmdWithPath); 349 if (isDebugging) 350 log.debug("copy cmd=" + cmdWithPath); 351 try { 352 exitCode = localSshObj.executeCmd(cmdWithPath, streamOut, 353 streamErr, destDetails.toString()); 354 } catch (ExecException e) { 355 return new CopyResult(1, e.toString(), ""); 356 } 357 if (exitCode > 0) { 358 log.error("Output on stdout:" + streamOut); 359 log.error("Output on stderr:" + streamErr); 360 } 361 String message = streamErr.toString(); 362 if (message == null || message.trim().equals("")) { 363 message = streamOut.toString(); 364 } 365 return new CopyResult(exitCode, message, warn.toString()); 366 } 367 368 /* 369 * Generates the bbcp command to copy from a remote host to another remote 370 * host. Executes the command using SshExec class 371 * 372 * @see 373 * org.sdm.spa.actors.transport.FileCopier#copyRemote(org.sdm.spa.actors 374 * .transport.vo.ConnectionDetails, java.lang.String, 375 * org.sdm.spa.actors.transport.vo.ConnectionDetails, java.lang.String, 376 * boolean) 377 */ 378 @Override 379 protected CopyResult copyRemote(ConnectionDetails srcDetails, 380 String srcFile, ConnectionDetails destDetails, String destFile, 381 boolean recursive) throws ExecException { 382 cmdLineOptions = ""; // Review - don't force overwrite***********#DONE 383 if (isDebugging) 384 log.debug("remote bbcp copy"); 385 OutputStream cmdStdout = new ByteArrayOutputStream(); 386 OutputStream cmdStderr = new ByteArrayOutputStream(); 387 // OutputStream streamOut = new ByteArrayOutputStream(); 388 // OutputStream streamErr = new ByteArrayOutputStream(); 389 String remoteHostStr = ""; 390 SshExec sshObjectSrc = null; 391 SshExec sshObjectDest = null; 392 // File single_srcFileObj = null; 393 // File srcFile_Obj = null; 394 // File srcFileObjlst = null; 395 // boolean flag = false; 396 String[] srcFileList = null; 397 String[] srcFile_list = null; 398 String wildcardFileList = ""; 399 StringBuffer warn = new StringBuffer(100); 400 boolean warn_flag = false; 401 402 // bbcp needs pseudo terminal enabled, so that password request is sent 403 // to stdout instead of terminal 404 // source connection object 405 sshObjectSrc = new SshExec(srcDetails.getUser(), srcDetails.getHost(), 406 srcDetails.getPort()); 407 sshObjectSrc.setTimeout(timeout, false, false); 408 sshObjectSrc.setForcedCleanUp(forcedCleanup); 409 sshObjectSrc.setPseudoTerminal(true); 410 411 // destination connection object 412 sshObjectDest = new SshExec(destDetails.getUser(), destDetails 413 .getHost(), destDetails.getPort()); 414 sshObjectDest.setTimeout(timeout, false, false); 415 sshObjectDest.setForcedCleanUp(forcedCleanup); 416 sshObjectDest.setPseudoTerminal(true); 417 418 if (srcDetails.isConnectionOrigin()) { 419 remoteHostStr = destDetails.toString(); 420 } else { 421 remoteHostStr = srcDetails.toString(); 422 } 423 424 StringBuffer cmd = new StringBuffer(100); 425 String cmdWithPath = null; 426 int exitCode = 0; 427 428 if (srcFile.contains(",")) { 429 // list of files 430 srcFileList = srcFile.split(","); 431 srcFile_list = new String[srcFileList.length]; 432 for (int count = 0; count < srcFileList.length; count++) { 433 if ((srcFileList[count].trim()).startsWith("/")) { 434 srcFile_list[count] = srcFileList[count].trim(); 435 } else { 436 warn.append(srcFileList[count].trim() + " "); 437 if (!warn_flag) 438 warn_flag = true; 439 } 440 } 441 if (warn_flag) { 442 warn 443 .append(" does not contain full path to the source file. Please provide full path. "); 444 warn_flag = false; 445 } 446 } else { 447 // single file 448 if (srcFile.startsWith("/")) { 449 srcFile_list = new String[1]; 450 srcFile_list[0] = srcFile.trim(); 451 } else { 452 throw new SshException( 453 srcFile 454 + "does not contain full path to the file. Please provide full path."); 455 } 456 } 457 // build bbcp command 458 try { 459 cmd.append("bbcp "); 460 cmd.append(cmdLineOptions); 461 if (!srcDetails.isConnectionOrigin()) { 462 // -z option for destination to source copy 463 cmd.append(" -z "); 464 } 465 // if (recursive) { 466 // BBCP does not handle error when directory is specified without 467 // recursive flag. 468 // Better to have recursive always on, rather than giving out wrong 469 // output. 470 cmd.append("-r "); 471 // } 472 log.debug("Protocol path src =" + protocolPathSrc); 473 if (!protocolPathSrc.equals("")) { 474 cmd.append("-S \"ssh -l %U %H "); 475 cmd.append(protocolPathSrc); 476 cmd.append("bbcp\" "); 477 } 478 if (!protocolPathDest.equals("")) { 479 cmd.append("-T \"ssh -l %U %H "); 480 cmd.append(protocolPathDest); 481 cmd.append("bbcp\" "); 482 } 483 if (srcDetails.isConnectionOrigin()) { 484 // BBCP can handle wildcard pattern * if copy is done 485 // from source machine to destination machine 486 for (int i = 0; i < srcFile_list.length; i++) { 487 if (srcFile_list[i].contains("*") 488 || (srcFile_list[i].contains("+"))) { 489 cmd.append(srcFile_list[i]); 490 cmd.append(" "); 491 } else { 492 cmd.append("\""); 493 cmd.append(srcFile_list[i]); 494 cmd.append("\""); 495 cmd.append(" "); 496 } 497 } 498 cmd.append(destDetails.toString()); 499 cmd.append(":"); 500 cmd.append(destFile); 501 } else { 502 for (int i = 0; i < srcFile_list.length; i++) { 503 // BBCP cannot handle wildcard pattern * if copy is 504 // being done from destination to source. We need 505 // function to list files matching the pattern 506 if (srcFile_list[i].contains("*") 507 || (srcFile_list[i].contains("+"))) { 508 wildcardFileList = sshObjectSrc 509 .getwildcardFileListingBBCP(srcDetails 510 .toString(), srcFile_list[i]); 511 cmd.append(wildcardFileList); 512 } else { 513 cmd.append(srcDetails.toString()); 514 cmd.append(":"); 515 cmd.append("\""); 516 cmd.append(srcFile_list[i]); 517 cmd.append("\""); 518 cmd.append(" "); 519 } 520 } 521 // cmd.append(destDetails.toString()); 522 // cmd.append(":"); 523 cmd.append(destFile); 524 } 525 if (protocolPathDest.equals("")) { 526 cmdWithPath = getCmdWithDefaultPath(cmd); 527 } else { 528 cmdWithPath = protocolPathDest + cmd; 529 } 530 531 log.debug("copy cmd without default path=" + cmd); 532 if (isDebugging) 533 log.debug("remote copy cmd=" + cmdWithPath); 534 535 System.out.println("cmdwithpath : " + cmdWithPath); 536 if (srcDetails.isConnectionOrigin()) { 537 exitCode = sshObjectSrc.executeCmd(cmdWithPath, cmdStdout, 538 cmdStderr, remoteHostStr); 539 } else { 540 exitCode = sshObjectDest.executeCmd(cmdWithPath, cmdStdout, 541 cmdStderr, remoteHostStr); 542 } 543 544 } catch (ExecException e) { 545 return new CopyResult(1, e.toString(), ""); 546 } 547 if (exitCode > 0) { 548 log.error("Output on stdout:" + cmdStdout); 549 log.error("Output on stderr:" + cmdStderr); 550 } 551 String message = cmdStderr.toString(); 552 if (message == null || message.trim().equals("")) { 553 message = cmdStdout.toString(); 554 } 555 return new CopyResult(exitCode, message, warn.toString()); 556 } 557 558 @Override 559 protected int getDefaultPort() { 560 return -1; 561 } 562 563}