001/** 002 * 003 * Copyright (c) 2010 The Regents of the University of California. 004 * All rights reserved. 005 * 006 * Permission is hereby granted, without written agreement and without 007 * license or royalty fees, to use, copy, modify, and distribute this 008 * software and its documentation for any purpose, provided that the 009 * above copyright notice and the following two paragraphs appear in 010 * all copies of this software. 011 * 012 * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY 013 * FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES 014 * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN 015 * IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY 016 * OF SUCH DAMAGE. 017 * 018 * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES, 019 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 020 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE 021 * PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY 022 * OF CALIFORNIA HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, 023 * UPDATES, ENHANCEMENTS, OR MODIFICATIONS. 024 */ 025 026package org.kepler.modulemanager; 027 028import java.io.BufferedInputStream; 029import java.io.BufferedOutputStream; 030import java.io.File; 031import java.io.FileInputStream; 032import java.io.FileNotFoundException; 033import java.io.FileOutputStream; 034import java.io.InputStream; 035import java.io.OutputStream; 036import java.net.HttpURLConnection; 037import java.net.URL; 038import java.util.ArrayList; 039import java.util.Iterator; 040import java.util.List; 041import java.util.Vector; 042import java.util.zip.ZipEntry; 043import java.util.zip.ZipInputStream; 044 045import org.apache.tools.ant.Project; 046import org.kepler.build.modules.Module; 047import org.kepler.build.modules.ModuleTree; 048import org.kepler.build.modules.ModulesTxt; 049import org.kepler.build.util.CommandLine; 050import org.kepler.build.util.HttpsSvnReader; 051 052/** 053 * Class to download modules when they're needed 054 * 055 * @author berkley 056 */ 057public class ModuleDownloader 058{ 059 private List<String> releasedModules = new ArrayList<String>(); 060 private Vector listeners = new Vector(); 061 062 /** 063 * constructor 064 */ 065 public ModuleDownloader() 066 { 067 calculateReleasedModules(); 068 listeners = new Vector(); 069 } 070 071 /** 072 * add a listener for ModuleManagerEvents 073 * @param listener 074 */ 075 public void addListener(ModuleManagerEventListener listener) 076 { 077 if(listeners == null) 078 { 079 listeners = new Vector(); 080 } 081 082 listeners.add(listener); 083 } 084 085 /** 086 * remove the given listener 087 * @param listener 088 */ 089 public void removeListener(ModuleManagerEventListener listener) 090 { 091 listeners.remove(listener); 092 } 093 094 /** 095 * download a set of modules as defined in a modules.txt file 096 * @param modulesTxt 097 */ 098 public void downloadFromModulesTxt(ModulesTxt modulesTxt) 099 throws Exception 100 { 101 ModuleTree tree = new ModuleTree(modulesTxt); 102 downloadModulesFromModuleList(tree.getModuleList()); 103 } 104 105 private void downloadModulesFromModuleList(List<Module> moduleList) 106 throws Exception 107 { 108 Vector v = new Vector(); 109 Iterator it = moduleList.iterator(); 110 while(it.hasNext()) 111 { 112 Module m = (Module)it.next(); 113 //System.out.println("adding module " + m.getName() + " to the list."); 114 v.add(m.getName()); 115 } 116 downloadModules((List)v); 117 } 118 119 /** 120 * download any modules in the list 121 * 122 * @param moduleNames the list of modules to download 123 */ 124 public void downloadModules(List<String> moduleNames) 125 throws Exception 126 { 127 Project project = new Project(); 128 for (String moduleName : moduleNames) 129 { 130 //split to throw away any trailing string, eg ptII repo 131 moduleName = moduleName.trim().split("\\s")[0]; 132 boolean isSuite = false; 133 if (moduleName.startsWith("*")) 134 { 135 isSuite = true; 136 moduleName = moduleName.substring(1, moduleName.length()); 137 } 138 if (moduleName.matches("[a-zA-Z-]+\\d+\\.\\d+")) 139 { 140 moduleName = moduleName + "." + getHighestPatch(moduleName); 141 } 142 else if (moduleName.matches("[a-zA-Z-]+\\d+\\.\\d+\\.\\^")) 143 { 144 moduleName = moduleName.substring(0, moduleName.length() - 2); 145 moduleName = moduleName + "." + getHighestPatch(moduleName); 146 } 147 148 Module module = Module.make(moduleName); 149 download(module); 150 unzip(module, project); 151 152 if (isSuite) 153 { 154 String url = RepositoryLocations.getReleaseLocation() + "/" + moduleName 155 + "/module-info/modules.txt"; 156 downloadModules(HttpsSvnReader.readURL(url)); 157 } 158 } 159 } 160 161 /** 162 * find the released modules 163 * 164 */ 165 public void calculateReleasedModules() 166 { 167 releasedModules = HttpsSvnReader.read(RepositoryLocations.getReleaseLocation()); 168 // XXX Update released.txt and Module.released here too to be in sync. 169 // TODO check about refactoring to get rid of releasedModules variable. 170 Module.updateReleasedModuleList(); 171 } 172 173 /** 174 * get the highest patch 175 * 176 * @param base 177 * @return 178 */ 179 private String getHighestPatch(String base) 180 { 181 try 182 { 183 if (base.startsWith("*")) 184 { 185 base = base.substring(1, base.length()); 186 } 187 int highestPatch = 0; 188 for (String branch : releasedModules) 189 { 190 if (branch.matches(base + "\\.\\d+")) 191 { 192 String[] parts = branch.split("\\."); 193 String patchString = parts[parts.length - 1]; 194 int patch = Integer.parseInt(patchString); 195 if (patch > highestPatch) 196 { 197 highestPatch = patch; 198 } 199 } 200 } 201 return "" + highestPatch; 202 } 203 catch (Exception e) 204 { 205 e.printStackTrace(); 206 return null; 207 } 208 } 209 210 /** 211 * download a single module 212 * 213 * @param module 214 */ 215 private void download(Module module) 216 throws Exception 217 { 218 int BUFFER = 1024; 219 File parentDir = module.getDir().getParentFile(); 220 File zip = new File(parentDir, module.getName() + ".zip"); 221 File target = new File(parentDir, module.getName()); 222 if (zip.exists() || target.isDirectory()) 223 { 224 return; 225 } 226 System.out.println("Downloading " + module.getName() + "..."); 227 String moduleLocation = RepositoryLocations.getReleaseLocation() + module.getName(); 228 String urlName = moduleLocation + "/" + zip.getName(); 229 //System.out.println("urlName:"+urlName); 230 URL url = new URL(urlName); 231 HttpURLConnection uc = (HttpURLConnection) url.openConnection(); 232 233 // check whether the url exists or not. 234 uc.setRequestMethod("HEAD"); 235 //System.out.println("uc.getResponseCode():" + uc.getResponseCode()); 236 if (uc.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 237 throw new FileNotFoundException (url + " was not found. \nPlease check the correctness of the suite/module and try again."); 238 } 239 //if exists, download it. 240 uc = (HttpURLConnection) url.openConnection(); 241 updateListenersDownloadBegin(uc.getContentLength(), module.getName()); 242 InputStream is = new BufferedInputStream(uc.getInputStream()); 243 OutputStream os = new BufferedOutputStream(new FileOutputStream(zip)); 244 byte data[] = new byte[BUFFER]; 245 int count = 0; 246 int numread = is.read(data, 0, BUFFER); 247 while (numread >= 0) 248 { 249 updateListenersProgress(uc.getContentLength(), BUFFER, count++, is); 250 os.write(data, 0, numread); 251 numread = is.read(data, 0, BUFFER); 252 } 253 os.close(); 254 is.close(); 255 256 // this may be a little expensive to call here every time, but it's safer. 257 Module.updatePresentModuleList(); 258 259 updateListenersDownloadEnd(); 260 261 } 262 263 /** 264 * update the listeners on the status of the download 265 * @param totalSize the total size of the transfer. -1 if not known 266 * @param bufferSize the size of each buffered transfer. -1 if this is not a 267 * buffered transfer 268 * @param readCount the number of times bufferSize has been read. 269 * @param is the inputStream being read 270 */ 271 private void updateListenersProgress(int totalSize, int bufferSize, int readCount, InputStream is) 272 { 273 for(int i=0; i<listeners.size(); i++) 274 { 275 ModuleManagerEventListener listener = (ModuleManagerEventListener)listeners.get(i); 276 listener.updateProgress(totalSize, bufferSize, readCount, is); 277 } 278 } 279 280 /** 281 * update the listeners when a download begins 282 * @param totalSize the total size of the download. -1 if unknown. 283 */ 284 private void updateListenersDownloadBegin(int totalSize, String moduleName) 285 { 286 for(int i=0; i<listeners.size(); i++) 287 { 288 ModuleManagerEventListener listener = (ModuleManagerEventListener)listeners.get(i); 289 listener.downloadBegin(totalSize, moduleName); 290 } 291 } 292 293 /** 294 * update the listeners when a download ends 295 */ 296 private void updateListenersDownloadEnd() 297 { 298 for(int i=0; i<listeners.size(); i++) 299 { 300 ModuleManagerEventListener listener = (ModuleManagerEventListener)listeners.get(i); 301 listener.downloadEnd(); 302 } 303 } 304 305 /** 306 * update the listeners on the status of the unzip 307 * @param totalSize the total size of the transfer. -1 if not known 308 * @param bufferSize the size of each buffered transfer. -1 if this is not a 309 * buffered transfer 310 * @param readCount the number of times bufferSize has been read. 311 * @param is the inputStream being read 312 */ 313 private void updateUnzipListenersProgress(long totalSize, int bufferSize, int readCount) 314 { 315 for(int i=0; i<listeners.size(); i++) 316 { 317 ModuleManagerEventListener listener = (ModuleManagerEventListener)listeners.get(i); 318 listener.unzipUpdateProgress(totalSize, bufferSize, readCount); 319 } 320 } 321 322 /** 323 * update the listeners when an unzip begins 324 * @param totalSize the total size of the download. -1 if unknown. 325 */ 326 private void updateListenersUnzipBegin(long totalSize, String moduleName) 327 { 328 for(int i=0; i<listeners.size(); i++) 329 { 330 ModuleManagerEventListener listener = (ModuleManagerEventListener)listeners.get(i); 331 listener.unzipBegin(totalSize, moduleName); 332 } 333 } 334 335 /** 336 * update the listeners when an unzip ends 337 */ 338 private void updateListenersUnzipEnd() 339 { 340 for(int i=0; i<listeners.size(); i++) 341 { 342 ModuleManagerEventListener listener = (ModuleManagerEventListener)listeners.get(i); 343 listener.unzipEnd(); 344 } 345 } 346 347 /** 348 * unzip a module for a project 349 * 350 * @param module 351 * @param project 352 */ 353 private void unzip(Module module, Project project) 354 throws Exception 355 { 356 final int BUFFER = 2048; 357 File parentDir = module.getDir().getParentFile(); 358 File zip = new File(parentDir, module.getName() + ".zip"); 359 updateListenersUnzipBegin(zip.length(), module.getName()); 360 File target = new File(parentDir, module.getName()); 361 if (target.isDirectory()) { 362 return; 363 } 364 365 final File moduleDir = new File(parentDir, module.getName()); 366 367 try (FileInputStream fis = new FileInputStream(zip); 368 ZipInputStream zis = new ZipInputStream(new BufferedInputStream(fis));) { 369 ZipEntry entry; 370 while ((entry = zis.getNextEntry()) != null) 371 { 372 final File outputFile = new File(moduleDir, entry.getName()); 373 if (entry.isDirectory()) 374 { 375 outputFile.mkdirs(); 376 continue; 377 } 378 int count = 0; 379 byte[] data = new byte[BUFFER]; 380 try(FileOutputStream fos = new FileOutputStream(outputFile); 381 BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER);) { 382 int c = 0; 383 while ((count = zis.read(data, 0, BUFFER)) != -1) 384 { 385 updateUnzipListenersProgress(zip.length(), BUFFER, c++); 386 dest.write(data, 0, count); 387 } 388 dest.flush(); 389 } 390 } 391 updateListenersUnzipEnd(); 392 393 File postInstallFile; 394 if(System.getProperty("os.name").startsWith("Windows")) { 395 postInstallFile = new File(moduleDir, "postInstall.bat"); 396 } else { 397 postInstallFile = new File(moduleDir, "postInstall.sh"); 398 } 399 400 if(postInstallFile.exists()) { 401 System.out.println("Running post install file " + postInstallFile); 402 // set it executable 403 if(!postInstallFile.setExecutable(true)) { 404 System.err.println("ERROR: could not make executable: " + postInstallFile); 405 } else { 406 // execute it. 407 CommandLine.exec(new String[] { postInstallFile.getAbsolutePath() }, 408 postInstallFile.getParentFile()); 409 } 410 } 411 } 412 } 413}