package org.ngbw.pise.commandrenderer;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.ngbw.sdk.api.tool.CommandRenderer;
import org.ngbw.sdk.common.util.InputStreamCollector;
import org.ngbw.sdk.common.util.StringUtils;
import org.ngbw.sdk.common.util.SuperString;
import org.ngbw.sdk.tool.RenderedCommand;
/**
* This Class implements CommandRenderer this implementation takes care of
* PiseXML and Perl contents
*
* @author Rami
*
* @author R. Hannes Niedner
*
*/
public class PiseCommandRenderer implements CommandRenderer
{
private static Log log = LogFactory.getLog(PiseCommandRenderer.class .getName());
//JAXB marshaller with mapped beans in the pise subpackage keyed to the cfg file url
private final Map cfgMap = new HashMap();
//pise PiseMarshaller for the submitted cfgFileURL
private PiseMarshaller piseMarshaller;
//parameter map (parameter name -> parameter value)
private Map parameters;
// private variable for the toUnixCmd() method
private String[] unixCmdGroup = null;
private String[] unixCmdGroupNegative = null;
private Map inputDataMapTab = null;
private RenderedCommand renderedCommand= null;
private final static String SCHEDULER_CONF = "scheduler.conf";
private PerlEval perlEval = null;
public PiseCommandRenderer()
{
super();
init();
}
private void init()
{
parameters = null;
piseMarshaller = null;
unixCmdGroup = new String[100];
unixCmdGroupNegative = new String[100];
inputDataMapTab = new HashMap();
renderedCommand = new RenderedCommand();
}
/**
@param url Url of the pise xml file
@param parameters A map of parameter name to value, where parameter name is the name element
of a parameter element in the pise xml file and where value may be something like
"1" or "y", or the contents of a source document, depending on the type of parameter.
*/
public RenderedCommand render(URL url, Map parameters)
{
try
{
log.debug("render command from " + parameters.size() + " parameters using config:" + url);
init();
this.parameters = parameters;
piseMarshaller = initPiseMarshaller(url);
perlEval = new PerlEval();
perlEval.initialize();
List commandList = toUnixCmd();
perlEval.terminate();
log.debug("toUnixCmd returns a list with " + commandList.size() + " elements");
String[] commandArray = new String[commandList.size()];
renderedCommand.setCommand(commandList.toArray(commandArray));
if (log.isDebugEnabled())
{
log.debug("Returned command: " + StringUtils.join(renderedCommand.getCommand(), " "));
Map input = renderedCommand.getInputFileMap();
Map output = renderedCommand.getOutputFileMap();
for(String parameter : input.keySet())
log.debug("Input: " + parameter + ": " + input.get(parameter));
for(String parameter : output.keySet())
log.debug("Output: " + parameter + ": " + output.get(parameter));
}
setSchedulerProperties();
return renderedCommand;
}
catch (Exception err)
{
throw new RuntimeException(err);
}
finally
{
if (perlEval != null)
{
perlEval.cleanup();
}
}
}
/**
If there the xml specified a param file with a specific name (given by SCHEDULER_CONF)
we expect that file to contain properties for scheduling the job. We load the contents
of the file into the renderedCommand.schedulerProperties.
*/
private void setSchedulerProperties()
{
log.debug("Looking for " + SCHEDULER_CONF + " to load properties");
Map inputData = new TreeMap();
Map inputFileNames = renderedCommand.getInputFileMap();
String p = null;
for (Iterator> names = inputFileNames.entrySet().iterator() ; names.hasNext() ; )
{
Map.Entry entry = names.next();
String parameter = entry.getKey();
String fileName = entry.getValue();
if (fileName.equals(SCHEDULER_CONF))
{
log.debug("Found " + SCHEDULER_CONF + " in renderedCommand.inputFileMap, for parameter: " + parameter);
p = parameter;
break;
}
}
byte[] value;
if (p == null || (value = parameters.get(p)) == null)
{
log.debug("Parameter " + p + " value is null.");
return;
}
log.debug("Parameter " + p + " value is:" + new String(value));
try
{
log.debug("Loading properties from byte array");
renderedCommand.getSchedulerProperties().load(new ByteArrayInputStream(value));
}
catch (Throwable t)
{
log.warn("Error loading scheduler properties", t);
}
}
//initialize the JAXB Marshaller
private PiseMarshaller initPiseMarshaller(URL url)
{
if (url == null)
throw new NullPointerException("Tool config file URL is null!");
if (cfgMap.containsKey(url) == false)
try
{
PiseMarshaller pm = new PiseMarshaller(url.openStream());
cfgMap.put(url, pm);
} catch (IOException e)
{
throw new RuntimeException("Cannot initialize PiseMarshaller.", e);
}
return cfgMap.get(url);
}
//all parameter names in the submitted maps have an underscore appended
//thus we declare our own private getters and setters for the value
private Set getParameterSet()
{
Set keySet = new HashSet();
for (String key : parameters.keySet())
{
//the regex secefies the only a trailing _
//and the replaceFirst will only take this one
String paramName = key.replaceFirst("_$", "");
keySet.add(paramName);
}
return keySet;
}
private String getParameterKey(String parameter)
{
return parameter + "_";
}
private String getParameterValue(String parameter)
{
String key = getParameterKey(parameter);
if (parameters.containsKey(key) || parameters.get(key) != null)
return new String(parameters.get(key));
else
return null;
// that is expected since parameters with
// null values are notin the parameter map
}
private void setParameterValue(String parameter, String value)
{
String key = parameter + "_";
parameters.put(key, value.getBytes());
}
private void setInputFileName(String parameter, String fileName)
{
String key = parameter + "_";
renderedCommand.getInputFileMap().put(key, fileName);
}
private void setOutputFileName(String parameter, String fileName)
{
renderedCommand.getOutputFileMap().put(parameter, fileName);
}
private String evaluatePerlStatement(String perlStatement) throws Exception
{
return perlEval.evaluateStatement(perlStatement);
}
/*
private String evaluatePerlStatement(String perlStatement) throws IOException, InterruptedException, ExecutionException
{
log.debug("PerlStatement: " + perlStatement);
// after some test, we had to create a table containing the different element of
// the statement, otherwise it seems not working with one String
String[] command = new String[3];
command[0] = "perl";
command[1] = "-e";
command[2] = perlStatement;
Process worker = Runtime.getRuntime().exec(command);
final Future stdOut = InputStreamCollector.readInputStream(worker.getInputStream());
final Future stdErr = InputStreamCollector.readInputStream(worker.getErrorStream());
final int exitCode = worker.waitFor();
if (exitCode != 0)
throw new RuntimeException("Exit value was not 0 but " + exitCode + " STDERR: \n" + stdErr.get());
return stdOut.get().trim();
}
*/
private List toUnixCmd() throws IOException, InterruptedException, ExecutionException,
Exception
{
//String Cmd = null;
List commandList = new ArrayList();
//1- A command is constructed from multiple type of parameters:
// These are the parameters passed as Map argument for this class, which
// means parameter selected through GUI
log.debug("toUnixCmd: processing " + getParameterSet().size() + " GUI parameters");
for (String paramName : getParameterSet())
processParameter(paramName, false);
// These are the parameters hidden from the GUI, but necessary for
// generating the command line correctly
log.debug("toUnixCmd: processing " + piseMarshaller.getHiddenSet().size() + " HIDDEN parameters");
for (String paramName : piseMarshaller.getHiddenSet())
processParameter(paramName, true);
// These are the parameters that generate an outfile, they are hidden
// from the GUI since the GUI dosen't
// give the user the possibility to specify the names of the output
// files, we use a standard filenames for each output file
// we have added this information (filenames) into the Pise XML file
// we return these parameter through getOutputFileMap()
log.debug("toUnixCmd: processing " + piseMarshaller.getOutfileSet().size() + " OUTFILESET parameters");
for (String paramName : piseMarshaller.getOutfileSet())
processParameter(paramName, false);
// These are the parameters passed as Set argument for this class, which
// mean that the user has selected a User Data Item
// for each of these parameters. we will return these parameters through
// getInputFileMap()
// if the file is a parameter file, the value of the parameter file is
// returned through getInputDataMap()
// processInfile_SequenceParameters();
// similar to outfile, one difference is not all output files are
// defined as outfile but also as Results
log.debug("toUnixCmd: processing " + piseMarshaller.getResultSet().size() + " RESULTSET parameters");
for (String paramName : piseMarshaller.getResultSet())
processParameter(paramName, true);
/*
2- Arranging inputDataMapTab into inputDataMap
inputDataMapTab is needed since we have to order the values with respect
to the Group definition in Pise XML files. we also add the paramfile inside the inputfileMap
*/
for(String key : inputDataMapTab.keySet())
{
String[] values = inputDataMapTab.get(key);
StringBuilder valueSb = new StringBuilder();
for(String value : values)
if (value != null)
valueSb.append(value);
String value = valueSb.toString();
// This replaces the two character sequence of with a single newline character.
// Since evalutePerl() returns the stdtout trimmed of leading and trailing whitespace, if we
// want to have a parameter be followed by a newline, we need the result of the perl evaluation
// to end not with a newline character but with a backslash followed by an n, which we'll replace
// with a newline right here.
//
// note that when you print a java string, as the logging messages in this file do, if
// the string contains an actual newline, it will print on multiple lines. If you see
// "\n" in the value displayed, it means the string contains two characters: a backslash
// followed b an n. This is mostly what we see because if you put "\n" in an xml file
// element, the xml unmarshaller delivers this as a java string containing a backslash
// followed by an n.
//
// Both perl (if using double quoted strings) and java interpret "\\n" as the two character
// sequence . The first quote escapes the second, yielding a literal
// backslash and the n is unquoted. In a perl single quoted string, '\\n' is three characters:
// two backslashes followed by an n. If you have a format like '$value\\n', with single quotes,
// perl leaves the \\n alone and when we get here we replace the \n with a newline, so we're
// left with a single backslash followed by a newline ... probably not what you want.
// On the other hand, "$value\\n" works because when ask perl to print this it gives us
// the value followed by a backslash follwed by an n.
setParameterValue(key, value.replace("\\n", "\n"));
setInputFileName(key, key);
}
// 3- Creating the command line by ordering them with respect to their
// group
if (unixCmdGroup[0] != null)
{
commandList.add(unixCmdGroup[0]);
log.debug("for unixCmdGroup[0] commandList.add: " + unixCmdGroup[0]);
}
for (int j = 1; j < unixCmdGroup.length; j++)
{
// Arranging positve groups
if (unixCmdGroup[j] != null)
{
commandList.add(unixCmdGroup[j]);
log.debug("for unixCmdGroup[" + j + "] commandList.add: " + unixCmdGroup[j]);
}
}
// Arranging the negative groups
for (int k=1 ; k '" + value[groupValue] + "'");
} else
{
if (groupValue >= 0)
{
if (unixCmdGroup[groupValue] != null)
unixCmdGroup[groupValue] += " " + perlFormat;
else
unixCmdGroup[groupValue] = perlFormat;
log.debug("unixCmdGroup[" + groupValue + "] = " + unixCmdGroup[groupValue]);
} else
{
if (unixCmdGroupNegative[Math.abs(groupValue)] != null)
unixCmdGroupNegative[Math.abs(groupValue)] += " " + perlFormat;
else
unixCmdGroupNegative[Math.abs(groupValue)] = perlFormat;
log.debug("unixCmdGroupNegative[" + Math.abs(groupValue) + "] = " + unixCmdGroupNegative[Math.abs(groupValue)]);
}
}
}
if (filenames != null)
{
// filenames not null for infile and sequence means we have assigned
// standard names for these input files
if (type.equals("InFile") || type.equals("Sequence"))
{
setInputFileName(paramName, filenames);
log.debug("setInputFileName: '" + paramName + "' -> '" + filenames + "'");
}
// Most output filenames are given in Results
// but can also be defined in Switch, List,
// OutFile, etc.
else
{
setOutputFileName(paramName, filenames);
log.debug("setOutputFileName: '" + paramName + "' -> '" + filenames + "'");
}
}
} else
{
log.debug("perlPrecond: " + (perlPrecond == null ? "null" : perlPrecond));
log.debug("perlFormat: " + (perlFormat == null ? "null" : perlFormat));
// evaluate precond before adding an element to the CMD
if (perlPrecond != null)
{
log.debug("Boolean.valueOf(perlPrecond.trim()) == " + Boolean.valueOf(perlPrecond));
if (Boolean.valueOf(perlPrecond))
{
if (perlFormat != null)
{
// check if should be written to a file or to cmd
log.debug("paramfile: " + (paramfile == null ? "null" : paramfile));
if (paramfile != null)
{
String[] value;
if (inputDataMapTab.containsKey(paramfile) == false || inputDataMapTab.get(paramfile) == null)
inputDataMapTab.put(paramfile, new String[100]);
value = inputDataMapTab.get(paramfile);
if (value[groupValue] != null)
value[groupValue] += perlFormat;
else
value[groupValue] = perlFormat;
} else
{
if (groupValue >= 0)
{
if (unixCmdGroup[groupValue] != null)
unixCmdGroup[groupValue] += " "
+ perlFormat;
else
unixCmdGroup[groupValue] = perlFormat
.toString();
log.debug("unixCmdGroup[" + groupValue + "] = " + unixCmdGroup[groupValue]);
} else
{
if (unixCmdGroupNegative[Math.abs(groupValue)] != null)
unixCmdGroupNegative[Math.abs(groupValue)] += " " + perlFormat;
else
unixCmdGroupNegative[Math.abs(groupValue)] = perlFormat .toString();
log.debug("unixCmdGroupNegative[" + Math.abs(groupValue) + "] = " + unixCmdGroupNegative[Math.abs(groupValue)]);
}
}
}
// some filesnames are given in the any type such as Switch,
// list, etc.
if (filenames != null)
{
if (type.equals("InFile") == false
&& type.equals("Sequence") == false)
{
setOutputFileName(paramName, filenames);
log.debug("type = " + type + ": setOutputFileName: '" + paramName + "' -> '" + filenames + "'");
}
// this statement is redanduant with the preceding one if
// all the filenames of output are given in Results
if (type.equals("Results"))
{
setOutputFileName(paramName, filenames);
log.debug("type = " + type + ": setOutputFileName: '" + paramName + "' -> '" + filenames + "'");
}
// filenames not null for infile and sequence means we have
// assigned standard names for these input files
if (type.equals("InFile") || type.equals("Sequence"))
{
setInputFileName(paramName, filenames);
log.debug("type = " + type + ": setInputFileName: '" + paramName + "' -> '" + filenames + "'");
}
}
}
}
}
log.debug("END : processParameter: " + paramName + " (hidden: " + hidden + ")\n");
}
/*
TL: I'm adding this method to work around the problem where we want to know whether
the user uploaded a file, or pasted in some text, by checking to see if $param_name
is in the map. The problem with doing something like "($param_name)?...:..." is
that the restitution replaces $param_name with the contents of the file or pasted in
text, and if that contains a double quote or other character that has meaning to perl,
the resulting perl expression may be garbage. Instead we can say "(defined $param_name) ?
...:..." and this method will replace the conditional with 1 or 0.
- replaces "defined $value" with 1 (since in swami, $value is always defined)
- replaces "defined $vdef" with 1 if there is a vdef attribute in the pise xml element, 0 otherwise
- replaces "defined $param_name" with 1 if param_name is in the map of params we received from the gui,
0 otherwise.
*/
private String replaceDefined(String str, String parameterValue, String vdef)
{
StringBuffer buf = new StringBuffer();
// Find the text "defined", followed by whitespace, followed by "$", followed by one or more
// "word" characters (i.e letter, number or underscore). Capture the "word" into group 1.
Pattern p = Pattern.compile("defined\\s\\$(\\w+)");
Matcher matcher = p.matcher(str);
while(matcher.find())
{
// replacement text depends on the word that follows the $, that is, group(1).
// but we'll be replacing the full string that was matched, in other words "defined $something"
// will be replace with "0" or "1" depending on what $something is.
String var = matcher.group(1);
if (var.equals("value"))
{
matcher.appendReplacement(buf, (parameterValue == null) ? "0" : "1");
} else if (var.equals("vdef"))
{
matcher.appendReplacement(buf, (vdef == null) ? "0" : "1");
} else
{
boolean defined = getParameterSet().contains(var);
matcher.appendReplacement(buf, defined ? "1": "0");
}
}
matcher.appendTail(buf);
return buf.toString();
}
/*
Returns a perl statement which will be passed to "perl -e". The stdout from
running perl -e on the statement will produce the text "true" or "false". We pass
the stdout to java's Boolean.valueOf() to get a boolean evaluation of the precond.
*/
private String restitutionPrecond(String precond, String paramName, String vdef)
{
// find every substring in precond starting with $
// if the substring is $value replace it with the value of the parameter
// identified in paramMap with key
// if the substring is $x replace it with the value of the parameter
// identified in paramMap with x
log.debug("Original Precondition: " + precond);
String paramValue = getParameterValue(paramName);
precond = replaceDefined(precond, paramValue, vdef);
log.debug("Precondition after replaceDefined: " + precond);
Pattern p = Pattern.compile("\\$\\w*");
Matcher m = p.matcher("Precond: " + precond);
while (m.find())
{
if (m.group().contains("$value"))
precond = precond.replace("$value", "\"" + paramValue + "\"");
else if (m.group().contains("$vdef"))
precond = precond.replace("$vdef", "\"" + vdef + "\"");
else
// we should ignore restitution if the pattern match $
// preceding bug restituted $ by "null"
if (m.group().equalsIgnoreCase("$") == false)
{
String myKey = m.group().substring(1);
/*
In the special case where the code references $param_name and param_name isn't
in our map, we return "print ''", which when run, outputs nothing, which evaluates
to false.
*/
if (getParameterSet().contains(myKey) == false)
return "print \"\"";
precond = precond.replace(m.group(), "\""
+ getParameterValue(myKey)
+ "\"");
}
}
//In perl ("false")? print"true" : print "false"
// returns true
// so we are replacing all true by 1 and all false by 0
precond = precond.replaceAll("false", "0");
precond = precond.replaceAll("true", "1");
// precond returns a True or False depends on the condition being
// verified
precond = "(" + precond + ")? print \"true\" : print \"false\";";
log.debug("Restituted Precondition: " + precond);
return precond;
}
private String restitutionFormat(String format, String paramName, String vdef)
{
log.debug("Original Format: " + format);
String paramValue = getParameterValue(paramName);
format = replaceDefined(format, paramValue, vdef);
log.debug("Format after replaceDefined: " + format);
Pattern p = Pattern.compile("\\$\\w*");
Matcher m = p.matcher(format);
while (m.find())
{
if (m.group().contains("$value"))
{
//if (paramMap.containsKey(key) == false)
// throw new RuntimeException("Missing value for parameter: " + key);
format = format.replace("$value", paramValue);
} else if (m.group().contains("$vdef"))
{
//format = format.replace("$vdef", "\"" + vdef + "\"");
format = format.replace("$vdef", vdef);
} else
{
String myKey = m.group().substring(1);
String myValue = null;
if (getParameterSet().contains(myKey) == false)
{
//TL: modified for raxmlhpc.xml "model" parameter. Need to get default
//value of parameters that haven't been set in the gui but are referenced in this
//parameter's code element.
myValue = piseMarshaller.getVdef(myKey);
if (myValue == null) // no vDef element in the pise xml
{
//throw new RuntimeException("Missing value for parameter: " + myKey);
// or we could set myValue to "", or the old behavior was
// return "print \"\"";
myValue = "";
}
} else
{
myValue = getParameterValue(myKey);
}
format = format.replace(m.group(), myValue);
}
}
// A format can be a simple "..." or a condition (...)? "...":"..."
// If it's ? expression we stick a "print" in front of both components.
// But if it starts with "<<" we're going to assume the whole thing is a here document
// and not process the ? operator. This isn't exactly what a real perl interpreter
// would do but I think it's close enough for the pise and ngbw xml files we deal with these
// days.
if (!format.trim().startsWith("<<") && format.contains("?"))
{
format = format.substring(0, format.indexOf("?") + 1) + " print "
+ format.substring(format.indexOf("?") + 1);
format = format.substring(0, format.indexOf(":") + 1) + " print "
+ format.substring(format.indexOf(":") + 1);
} else
{
format = "print " + format;
}
format = format.replaceAll("false", "0");
format = format.replaceAll("true", "1");
log.debug("Restituted Format: " + format);
return format;
}
}