001/*
002 * Copyright (c) 2003-2012 The Regents of the University of California.
003 * All rights reserved.
004 *
005 * '$Author: barseghian $'
006 * '$Date: 2012-08-18 07:37:31 +0000 (Sat, 18 Aug 2012) $' 
007 * '$Revision: 30476 $'
008 * 
009 * Permission is hereby granted, without written agreement and without
010 * license or royalty fees, to use, copy, modify, and distribute this
011 * software and its documentation for any purpose, provided that the above
012 * copyright notice and the following two paragraphs appear in all copies
013 * of this software.
014 *
015 * IN NO EVENT SHALL THE UNIVERSITY OF CALIFORNIA BE LIABLE TO ANY PARTY
016 * FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
017 * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF
018 * THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF THE POSSIBILITY OF
019 * SUCH DAMAGE.
020 *
021 * THE UNIVERSITY OF CALIFORNIA SPECIFICALLY DISCLAIMS ANY WARRANTIES,
022 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
023 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE
024 * PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND THE UNIVERSITY OF
025 * CALIFORNIA HAS NO OBLIGATION TO PROVIDE MAINTENANCE, SUPPORT, UPDATES,
026 * ENHANCEMENTS, OR MODIFICATIONS.
027 *
028 */
029
030package org.ecoinformatics.seek.ecogrid;
031
032import java.io.StringReader;
033import java.io.StringWriter;
034import java.util.UUID;
035
036import javax.xml.parsers.DocumentBuilder;
037import javax.xml.parsers.DocumentBuilderFactory;
038import javax.xml.parsers.ParserConfigurationException;
039
040import org.apache.commons.logging.Log;
041import org.apache.commons.logging.LogFactory;
042import org.apache.xml.serialize.XMLSerializer;
043import org.apache.xpath.XPathAPI;
044import org.ecoinformatics.ecogrid.EcogridObjType;
045import org.ecoinformatics.ecogrid.client.AuthenticationServiceClient;
046import org.ecoinformatics.ecogrid.client.PutServiceClient;
047import org.ecoinformatics.seek.datasource.eml.eml2.Eml200Parser;
048import org.w3c.dom.Document;
049import org.w3c.dom.Node;
050import org.w3c.dom.NodeList;
051import org.xml.sax.InputSource;
052
053import ptolemy.actor.TypedAtomicActor;
054import ptolemy.actor.TypedIOPort;
055import ptolemy.data.ObjectToken;
056import ptolemy.data.StringToken;
057import ptolemy.data.expr.Parameter;
058import ptolemy.data.expr.StringParameter;
059import ptolemy.data.type.BaseType;
060import ptolemy.kernel.CompositeEntity;
061import ptolemy.kernel.util.Attribute;
062import ptolemy.kernel.util.IllegalActionException;
063import ptolemy.kernel.util.NameDuplicationException;
064import util.StaticUtil;
065
066/**
067 * The EcogridWriter actor writes a data file and the EML metadata describing that
068 * data file to a remote EcoGrid repositiory. Identifiers for the data and
069 * metadata are created and these IDs are sent to the output ports. These IDs
070 * might be used for future access to the data and metadata files. Note also
071 * that the ID of the data file is inserted into the metadata file as a
072 * reference (i.e., a pointer from the metadata to the data).
073 * 
074 * The EcoGrid is a distributed network providing scientists access to
075 * ecological, biodiversity, and environmental data and analytic resources. The
076 * EcoGrid can be used to store ecological data, or to model or analyze it via
077 * remote EcoGrid services.
078 * 
079 * Ecological Metadata Language (EML) is a standard set of terms and definitions
080 * used to describe ecological data. For example, EML metadata might contain
081 * infomation about a data set's units of measurement, date of collection,
082 * location, etc.
083 * 
084 * @author tao
085 */
086public class EcogridWriter extends TypedAtomicActor {
087
088
089        /**
090         * Accepts the file name and path of the local data file to upload to the EcoGrid service.
091         * WARNING: This port is ignored if you supply data to dataPort.
092         */
093        public TypedIOPort sourceFileNamePort = null;
094        
095        /**
096         * Can be used in lieu of supplying a file name to dataFileNamePort.
097         * The byte[] of the input object will be uploaded.
098         */
099        public TypedIOPort sourceDataPort;
100        
101        /**
102         * Accepts a string of metadata describing the data file. This string will be uploaded to Ecogrid service as metadata
103         */
104        public TypedIOPort metadataPort = null;
105        
106        /**
107         * An output port that broadcasts the metadata doc ID, which is generated by the actor for future reference.
108         */
109        public TypedIOPort metadataDocidPort = null;
110        
111        /**
112         * Broadcasts the data docid, which is generated by the actor for future reference.
113         */
114        public TypedIOPort dataDocidPort = null;
115
116        private String metadataDocid = null;
117        private String dataDocid = null;
118        private String metadataUserName = null;
119        private String metadataPasswd = null;
120        private String metadataDestination = null;
121
122        private String authenURL = null;
123        private String localDataFileName = null;
124        private byte[] localData;
125        private String metadataContent = null;
126
127        protected final static Log log;
128        protected final static boolean isDebugging;
129        static {
130                log = LogFactory
131                                .getLog("org.ecoinformatics.seek.ecogrid.EcoGridServicesController");
132                isDebugging = log.isDebugEnabled();
133        }
134
135        private String docIdSuffix = "doc";
136
137        private static final String DATAFILENAMEPORT = "dataFileNamePort";
138        private static final String DATAPORT = "dataPort";
139        private static final String METADATAPORT = "metadata";
140        private static final String METADATADOCIDPORT = "metadataDocid";
141        private static final String DATADOCIDPORT = "dataDocid";
142        private static final String METADATADESTINATION = "metadataDestination";
143        private static final String DISTRIBUTIONPATH = "physical/distribution/online/url";
144        private static final String SEPERATOR = ".";
145        private static final String ECOGRIDPROTOCOL = "ecogrid://knb/";
146        private static final String DEFAULTECOGRIDPUTSERVER = "http://ecogrid.ecoinformatics.org/knb/services/PutService";
147        private static final String DEFAULTECOGRIDAUTHENSERVER = "http://ecogrid.ecoinformatics.org/knb/services/AuthenticationService";
148        private static final String AUTHENURL = "authenticationURL";
149        private static final String USERNAME = "userName";
150        private static final String PASSWORD = "passWord";
151
152        /** Ecogrid service URL for receiving metadata and data */
153        public StringParameter metadataDesParam = null;
154        
155        /** Ecogrid service URL for authenticating user */
156        public StringParameter authenURLParam = null;
157        
158        /**
159         * User name for authenticatication. For example, it is a DN for knb ldap
160         * server. uid=smith,o=NCEAS,dc=eocinformatics,dc=org
161         */
162        public StringParameter usernameParam = null;
163        
164        /** Password for this user */
165        public StringParameter passwordParam = null;
166
167        // public FileParameter localDataFileNameParameter = null;
168
169        public EcogridWriter(CompositeEntity container, String name)
170                        throws IllegalActionException, NameDuplicationException {
171                super(container, name);
172                
173                // input ports
174                metadataPort = new TypedIOPort(this, METADATAPORT, true, false);
175                metadataPort.setMultiport(false);
176                metadataPort.setTypeEquals(BaseType.STRING);
177                
178                sourceFileNamePort = new TypedIOPort(this, DATAFILENAMEPORT, true,
179                                false);
180                sourceFileNamePort.setMultiport(false);
181                sourceFileNamePort.setTypeEquals(BaseType.STRING);
182                
183                sourceDataPort = new TypedIOPort(this, DATAPORT, true, false);
184            sourceDataPort.setTypeEquals(BaseType.OBJECT);
185
186                // output ports
187                metadataDocidPort = new TypedIOPort(this, METADATADOCIDPORT, false,
188                                true);
189                metadataDocidPort.setMultiport(false);
190                metadataDocidPort.setTypeEquals(BaseType.STRING);
191                dataDocidPort = new TypedIOPort(this, DATADOCIDPORT, false, true);
192                dataDocidPort.setMultiport(false);
193                dataDocidPort.setTypeEquals(BaseType.STRING);
194
195                // parameters
196                metadataDesParam = new StringParameter(this, METADATADESTINATION);
197                metadataDesParam.setExpression(DEFAULTECOGRIDPUTSERVER);
198                authenURLParam = new StringParameter(this, AUTHENURL);
199                authenURLParam.setExpression(DEFAULTECOGRIDAUTHENSERVER);
200                usernameParam = new StringParameter(this, USERNAME);
201                passwordParam = new StringParameter(this, PASSWORD);
202
203                // localDataFileNameParameter = new FileParameter(this,
204                // LOCATDATFILENAME);
205                _attachText("_iconDescription", "<svg>\n"
206                                + "<rect x=\"-25\" y=\"-20\" " + "width=\"50\" height=\"40\" "
207                                + "style=\"fill:white\"/>\n"
208                                + "<polygon points=\"-15,-10 -12,-10 -8,-14 -1,-14 3,-10"
209                                + " 15,-10 15,10, -15,10\" " + "style=\"fill:red\"/>\n"
210                                + "</svg>\n");
211        }
212
213        // /////////////////////////////////////////////////////////////////
214        // // public methods ////
215
216        /**
217         * If the specified attribute is <i>fileOrURL </i> and there is an open file
218         * being read, then close that file and open the new one; if the attribute
219         * is <i>numberOfLinesToSkip </i> and its value is negative, then throw an
220         * exception. In the case of <i>fileOrURL </i>, do nothing if the file name
221         * is the same as the previous value of this attribute.
222         * 
223         * @param attribute
224         *            The attribute that has changed.
225         * @exception IllegalActionException
226         *                If the specified attribute is <i>fileOrURL </i> and the
227         *                file cannot be opened, or the previously opened file
228         *                cannot be closed; or if the attribute is
229         *                <i>numberOfLinesToSkip </i> and its value is negative.
230         */
231
232        /**
233         * Determine the attribute changed value
234         * 
235         * @param attribute
236         *            The attribute that changed.
237         * @exception IllegalActionException
238         *                If the output type is not recognized.
239         */
240        public void attributeChanged(Attribute attribute)
241                        throws IllegalActionException {
242                if (attribute == metadataDesParam) {
243                        metadataDestination = getValueForAttributeChange(metadataDesParam);
244                } else if (attribute == authenURLParam) {
245                        authenURL = getValueForAttributeChange(authenURLParam);
246                } else if (attribute == usernameParam) {
247                        metadataUserName = getValueForAttributeChange(usernameParam);
248                } else if (attribute == passwordParam) {
249                        metadataPasswd = getValueForAttributeChange(passwordParam);
250                }
251        }
252
253        /**
254         * Get new value for attribute changes.
255         * 
256         * @param attribute
257         * @return
258         * @throws IllegalActionException
259         */
260        private String getValueForAttributeChange(Parameter attribute)
261                        throws IllegalActionException {
262
263                String newValue = null;
264                if (attribute != null) {
265                        StringToken token = (StringToken) attribute.getToken();
266                        if (token != null) {
267                                newValue = token.stringValue();
268                        }
269                }
270                if (isDebugging) {
271                        log.debug("The value of attribute is " + newValue);
272                }
273                // System.out.println("======the value of attribute is "+newValue);
274                return newValue;
275        }
276
277        /**
278         * @return Description of the Returned Value
279         * @exception IllegalActionException
280         *                Description of Exception
281         * @since
282         */
283        public boolean prefire() throws IllegalActionException {
284                int revision = 1;
285                
286                if(sourceDataPort.numberOfSources() > 0) {
287                    ObjectToken object = (ObjectToken) sourceDataPort.get(0);
288                    localData = (byte[]) object.getValue();
289                }       
290                else if (sourceFileNamePort.numberOfSources() > 0) {
291                // get data file name
292                        if (!sourceFileNamePort.hasToken(0)){
293                                return false;
294                        }
295                        StringToken sourcefnToken = (StringToken) sourceFileNamePort.get(0);
296                        localDataFileName = sourcefnToken.stringValue();
297                        // localDataFileNameParameter.setExpression(localDataFileName);
298                        if (isDebugging) {
299                                log.debug("The localDataFileName is " + localDataFileName);
300                        }
301                }
302                /*
303                 * else { localDataFileName =
304                 * localDataFileNameParameter.asFile().getPath(); }
305                 */
306
307                dataDocid = generateDocId(docIdSuffix, revision);
308                // get metadata
309                if (metadataPort.getWidth() > 0) {
310
311                        if (!metadataPort.hasToken(0)){
312                                return false;
313                        }
314                        StringToken metadataToken = (StringToken) metadataPort.get(0);
315                        metadataContent = metadataToken.stringValue();
316                        if (isDebugging) {
317                                log.debug("The original metadata is " + metadataContent);
318                        }
319                        // replace the url part
320                        String newURL = ECOGRIDPROTOCOL + dataDocid;
321                        metadataDocid = generateDocId(docIdSuffix, revision);
322                        try {
323                                metadataContent = replaceDistributionURLAndPackageID(metadataContent,
324                                                newURL, metadataDocid);
325                        } catch (Exception e) {
326                                throw new IllegalActionException(e.getMessage());
327                        }
328
329                }
330                
331                return super.prefire();
332        }
333
334        /**
335         * Output the data lines into an array.
336         * 
337         * @exception IllegalActionException
338         *                If there's no director.
339         */
340        public void fire() throws IllegalActionException {
341                super.fire();
342                try {
343                        long start = System.currentTimeMillis();
344                        // load metadata and data to ecogrid
345                        String sessionId = loginEcoGrid(authenURL, metadataUserName,
346                                        metadataPasswd);
347                        uploadDataFile(metadataDestination, localDataFileName, dataDocid, sessionId);
348                        uploadMetadata(metadataDestination, metadataContent, metadataDocid,
349                                        sessionId);
350                        long end = System.currentTimeMillis();
351                        System.out.println("**********EcogridWriter uploading data and metadata took "+(end-start)+" ms.");
352                        // output metadata docid and data docid
353                        TypedIOPort pp = (TypedIOPort) this.getPort(METADATADOCIDPORT);
354                        pp.send(0, new StringToken(metadataDocid));
355                        TypedIOPort pp1 = (TypedIOPort) this.getPort(DATADOCIDPORT);
356                        pp1.send(0, new StringToken(dataDocid));
357                        metadataDocid = null;
358                        dataDocid = null;
359                } catch (Exception e) {
360                        throw new IllegalActionException(e.getMessage());
361                }
362
363        }
364
365        /**
366         * This method will do login action and return a session id.
367         * 
368         * @param authernURL
369         * @param userName
370         * @param password
371         * @return
372         * @throws Exception
373         */
374        private String loginEcoGrid(String authernURL, String userName,
375                        String password) throws Exception {
376                String sessionId = null;
377                AuthenticationServiceClient client = new AuthenticationServiceClient(
378                                authernURL);
379                sessionId = client.login_action(userName, password);
380                if (isDebugging) {
381                        log.debug("The session id is " + sessionId);
382                }
383                // client.destory();
384                return sessionId;
385        }
386
387        /**
388         * Upload Data. 
389         * 
390         * If localData is not null, localFileName
391         * is ignored, you can set it null.
392         * 
393         * @param destURL
394         * @param localFileName
395         * @param docid
396         * @param sessionId
397         * @throws Exception
398         */
399        private void uploadDataFile(String destURL, String localFileName,
400                        String docid, String sessionId) throws Exception {
401                int type = EcogridObjType.DATA;
402                // client.createEcoGridPutLevelOnePortType();
403                byte[] data;
404                
405                if(localData != null) {
406                    data = localData;
407                }
408                else {
409                    data = StaticUtil.getBytesArrayFromFile(localFileName);
410                }
411        if(data.length < 10) {
412            System.out.println("WARNING: read " + data.length + " bytes in data file");
413        }
414
415        boolean error = true;
416        int tries = 5;
417        while(tries > 0 && error) {
418            tries--;
419            error = false;
420            PutServiceClient client = new PutServiceClient(destURL);
421                    try { 
422                client.put(data, docid, type, sessionId); 
423            }
424            catch(Exception e) { 
425                System.out.println("data exception: " + e.getMessage());
426                if(tries > 0) {
427                    error = true;
428                }
429                else {
430                    throw e;
431                }
432            }
433        }
434                // client.destroy();
435        }
436
437        /**
438         * Upload Metadata.
439         * 
440         * @param destURL
441         * @param metadataContent
442         * @param docid
443         * @param sessionId
444         * @throws Exception
445         */
446        private void uploadMetadata(String destURL, String metadataContent,
447                        String docid, String sessionId) throws Exception {
448                int type = EcogridObjType.METADATA;
449                byte[] content = metadataContent.getBytes();
450        if(content.length < 10) {
451            System.out.println("WARNING: read " + content.length + " bytes in metadata file");
452        }
453                PutServiceClient client = new PutServiceClient(destURL);
454                // client.createEcoGridPutLevelOnePortType();
455                try { 
456                        client.put(content, docid, type, sessionId);
457                }
458        catch(Exception e) { 
459            System.out.println("metadata exception: " + e.getMessage());
460            throw e;
461        }
462                // client.destroy();
463        }
464
465        /**
466         * After generate docid for data file, the original metadata need to replace
467         * the distribution url by new value. Currently we just consider eml as
468         * metadata.
469         * 
470         * @param originalMetadata
471         * @param newURL
472         * @param newMetadataID
473         * @return
474         * @throws Exception
475         */
476        private String replaceDistributionURLAndPackageID(String originalMetadata, String newURL, String newMetadataID)
477                        throws Exception {
478                String newMetadata = null;
479                if (originalMetadata != null) {
480                        DocumentBuilder parser = null;
481                        try {
482                                DocumentBuilderFactory factory = DocumentBuilderFactory
483                                                .newInstance();
484                                // factory.setNamespaceAware(true);
485                                parser = factory.newDocumentBuilder();
486                                if (parser == null) {
487                                        throw new Exception("Could not create Document parser in "
488                                                        + "EcogridWriter");
489                                }
490                        } catch (ParserConfigurationException pce) {
491                                throw new Exception("Could not create Document parser in "
492                                                + "EcogridWriter: " + pce.getMessage());
493                        }
494                        log.debug("after generate dom parser");
495                        Document doc = null;
496                        StringReader reader = new StringReader(originalMetadata);
497                        InputSource in = new InputSource(reader);
498                        log.debug("after generate inputsource");
499                        doc = parser.parse(in);
500                        log.debug("after parsing inputsource");
501                        // we assuming the metadata only have one entity
502                        String tablePath = Eml200Parser.TABLEENTITY + "/"
503                                        + DISTRIBUTIONPATH;
504                        NodeList tableNodeList = XPathAPI.selectNodeList(doc, tablePath);
505                        String rasterPath = Eml200Parser.SPATIALRASTERENTITY + "/"
506                                        + DISTRIBUTIONPATH;
507                        NodeList rasterNodeList = XPathAPI.selectNodeList(doc, rasterPath);
508                        String vectorPath = Eml200Parser.SPATIALVECTORENTITY + "/"
509                                        + DISTRIBUTIONPATH;
510                        NodeList vectorNodeList = XPathAPI.selectNodeList(doc, vectorPath);
511                        String procedurePath = Eml200Parser.STOREDPROCEDUREENTITY + "/"
512                                        + DISTRIBUTIONPATH;
513                        NodeList procedureNodeList = XPathAPI.selectNodeList(doc,
514                                        procedurePath);
515                        String viewPath = Eml200Parser.VIEWENTITY + "/" + DISTRIBUTIONPATH;
516                        NodeList viewNodeList = XPathAPI.selectNodeList(doc, viewPath);
517                        String otherEntityPath = Eml200Parser.OTHERENTITY + "/"
518                                        + DISTRIBUTIONPATH;
519                        NodeList otherEntityNodeList = XPathAPI.selectNodeList(doc,
520                                        otherEntityPath);
521                        if (tableNodeList != null && tableNodeList.getLength() != 0) {
522                                // have tableEntity
523                                log.debug("in data table path for replacement");
524                                setNewValueForNode(tableNodeList, newURL);
525                        } else if (rasterNodeList != null
526                                        && rasterNodeList.getLength() != 0) {
527                                setNewValueForNode(rasterNodeList, newURL);
528                        } else if (vectorNodeList != null
529                                        && vectorNodeList.getLength() != 0) {
530                                setNewValueForNode(vectorNodeList, newURL);
531                        } else if (procedureNodeList != null
532                                        && procedureNodeList.getLength() != 0) {
533                                setNewValueForNode(procedureNodeList, newURL);
534                        } else if (viewNodeList != null && viewNodeList.getLength() != 0) {
535                                setNewValueForNode(viewNodeList, newURL);
536                        } else if (otherEntityNodeList != null
537                                        && otherEntityNodeList.getLength() != 0) {
538                                setNewValueForNode(otherEntityNodeList, newURL);
539                        }
540                        
541                        //replace package id
542                        String packagePath= "/*[local-name() = '"+Eml200Parser.EML+"']/@"+Eml200Parser.PACKAGEID;
543                        NodeList packageIDNodeList = XPathAPI.selectNodeList(doc, packagePath);
544                        setNewValueForAttribute(packageIDNodeList, newMetadataID);
545                        // serialize the DOM tree
546                        StringWriter writer = new StringWriter();
547                        XMLSerializer serializer = new XMLSerializer();
548                        serializer.setOutputCharStream(writer);
549                        // serializer.setOutputByteStream(System.out);
550                        serializer.serialize(doc);
551                        newMetadata = writer.toString();
552                        // writer.write(newMetadata);
553                        log.debug("The new metadata with new data reference is \n"
554                                        + newMetadata);
555
556                }
557                return newMetadata;
558        }
559
560        /**
561         * Docid will look like suffix.id.rev. 
562         * Where id is a concat of currenttTimeMillis + random UUID
563         * 
564         * @param suffix
565         * @param rev
566         * @return
567         */
568        private String generateDocId(String suffix, int rev) {
569                //double random = Math.random();
570                //int randomInt = (new Double(random*1000000)).intValue();
571                String docid = null;
572                //Date currentTime = new Date();
573                //String id = Long.toString(currentTime.getTime());
574                //String id = Long.toString(System.currentTimeMillis());
575                //id= id+randomInt;
576                String id = Long.toString(System.currentTimeMillis());
577            id = id.concat(UUID.randomUUID().toString());
578                docid = suffix + SEPERATOR + id + SEPERATOR + rev;
579                log.debug("The generated docid is " + docid);
580                return docid;
581        }
582
583        /**
584         * This method will set up new value for the list. We only replace the first
585         * one.
586         * 
587         * @param list
588         * @param newValue
589         */
590        private void setNewValueForNode(NodeList list, String newValue) {
591                Node cn = list.item(0).getFirstChild();
592                if ((cn != null) && (cn.getNodeType() == Node.TEXT_NODE)) {
593                        log.debug("set new value " + newValue + " for distribution url");
594                        cn.setNodeValue(newValue);
595                }       
596        }
597        
598        /**
599         * Set a new value for an attribute node.
600         * 
601         * @param list
602         * @param newValue
603         */
604        private void setNewValueForAttribute(NodeList list, String newValue){
605          if(list != null && list.getLength() >0){
606            Node cn = list.item(0);
607            if(cn != null && cn.getNodeType() == Node.ATTRIBUTE_NODE){
608              //System.out.println("Set new value "+newValue +" for attribute"+cn.getNodeName());
609              cn.setNodeValue(newValue);
610            }
611          }
612        }
613        
614        
615
616}