/*******************************************************************************
 * Copyright (c) 2001-2005 Sasa Markovic and Ciaran Treanor.
 * Copyright (c) 2011 The OpenNMS Group, Inc.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 *******************************************************************************/

package org.jrobin.data;

import org.jrobin.core.*;

import java.io.IOException;
import java.util.*;

/**
 * Class which should be used for all calculations based on the data fetched from RRD files. This class
 * supports ordinary DEF datasources (defined in RRD files), CDEF datasources (RPN expressions evaluation),
 * SDEF (static datasources - extension of JRobin) and PDEF (plottables, see
 * {@link Plottable Plottable} for more information.<p>
 * <p/>
 * Typical class usage:<p>
 * <pre>
 * final long t1 = ...
 * final long t2 = ...
 * DataProcessor dp = new DataProcessor(t1, t2);
 * // DEF datasource
 * dp.addDatasource("x", "demo.rrd", "some_source", "AVERAGE");
 * // DEF datasource
 * dp.addDatasource("y", "demo.rrd", "some_other_source", "AVERAGE");
 * // CDEF datasource, z = (x + y) / 2
 * dp.addDatasource("z", "x,y,+,2,/");
 * // ACTION!
 * dp.processData();
 * // Dump calculated values
 * System.out.println(dp.dump());
 * </pre>
 */
public class DataProcessor implements ConsolFuns {
	/**
	 * Constant representing the default number of pixels on a JRobin graph (will be used if
	 * no other value is specified with {@link #setStep(long) setStep()} method.
	 */
	public static final int DEFAULT_PIXEL_COUNT = 600;
	private static final double DEFAULT_PERCENTILE = 95.0; // %

	private int pixelCount = DEFAULT_PIXEL_COUNT;

	/**
	 * Constant that defines the default {@link RrdDbPool} usage policy. Defaults to <code>false</code>
	 * (i.e. the pool will not be used to fetch data from RRD files)
	 */
	public static final boolean DEFAULT_POOL_USAGE_POLICY = false;
	private boolean poolUsed = DEFAULT_POOL_USAGE_POLICY;

	private final long tStart;
	private long tEnd, timestamps[];
	private long lastRrdArchiveUpdateTime = 0;
	// this will be adjusted later
	private long step = 0;
	// resolution to be used for RRD fetch operation
	private long fetchRequestResolution = 1;

	// the order is important, ordinary HashMap is unordered
	private Map<String, Source> sources = new LinkedHashMap<String, Source>();

	private Def[] defSources;

	/**
	 * Creates new DataProcessor object for the given time span. Ending timestamp may be set to zero.
	 * In that case, the class will try to find the optimal ending timestamp based on the last update time of
	 * RRD files processed with the {@link #processData()} method.
	 *
	 * @param t1 Starting timestamp in seconds without milliseconds
	 * @param t2 Ending timestamp in seconds without milliseconds
	 * @throws RrdException Thrown if invalid timestamps are supplied
	 */
	public DataProcessor(long t1, long t2) throws RrdException {
		if ((t1 < t2 && t1 > 0 && t2 > 0) || (t1 > 0 && t2 == 0)) {
			this.tStart = t1;
			this.tEnd = t2;
		}
		else {
			throw new RrdException("Invalid timestamps specified: " + t1 + ", " + t2);
		}
	}

	/**
	 * Creates new DataProcessor object for the given time span. Ending date may be set to null.
	 * In that case, the class will try to find optimal ending date based on the last update time of
	 * RRD files processed with the {@link #processData()} method.
	 *
	 * @param d1 Starting date
	 * @param d2 Ending date
	 * @throws RrdException Thrown if invalid timestamps are supplied
	 */
	public DataProcessor(Date d1, Date d2) throws RrdException {
		this(Util.getTimestamp(d1), d2 != null ? Util.getTimestamp(d2) : 0);
	}

	/**
	 * Creates new DataProcessor object for the given time span. Ending date may be set to null.
	 * In that case, the class will try to find optimal ending date based on the last update time of
	 * RRD files processed with the {@link #processData()} method.
	 *
	 * @param gc1 Starting Calendar date
	 * @param gc2 Ending Calendar date
	 * @throws RrdException Thrown if invalid timestamps are supplied
	 */
	public DataProcessor(Calendar gc1, Calendar gc2) throws RrdException {
		this(Util.getTimestamp(gc1), gc2 != null ? Util.getTimestamp(gc2) : 0);
	}

	/**
	 * Returns boolean value representing {@link org.jrobin.core.RrdDbPool RrdDbPool} usage policy.
	 *
	 * @return true, if the pool will be used internally to fetch data from RRD files, false otherwise.
	 */
	public boolean isPoolUsed() {
		return poolUsed;
	}

	/**
	 * Sets the {@link org.jrobin.core.RrdDbPool RrdDbPool} usage policy.
	 *
	 * @param poolUsed true, if the pool should be used to fetch data from RRD files, false otherwise.
	 */
	public void setPoolUsed(boolean poolUsed) {
		this.poolUsed = poolUsed;
	}

	/**
	 * Sets the number of pixels (target graph width). This number is used only to calculate pixel coordinates
	 * for JRobin graphs (methods {@link #getValuesPerPixel(String)} and {@link #getTimestampsPerPixel()}),
	 * but has influence neither on datasource values calculated with the
	 * {@link #processData()} method nor on aggregated values returned from {@link #getAggregates(String)}
	 * and similar methods. In other words, aggregated values will not change once you decide to change
	 * the dimension of your graph.<p>
	 * <p/>
	 * The default number of pixels is defined by constant {@link #DEFAULT_PIXEL_COUNT}
	 * and can be changed with a {@link #setPixelCount(int)} method.
	 *
	 * @param pixelCount The number of pixels. If you process RRD data in order to display it on the graph,
	 *                   this should be the width of your graph.
	 */
	public void setPixelCount(int pixelCount) {
		this.pixelCount = pixelCount;
	}

	/**
	 * Returns the number of pixels (target graph width). See {@link #setPixelCount(int)} for more information.
	 *
	 * @return Target graph width
	 */
	public int getPixelCount() {
		return pixelCount;
	}

	/**
	 * Roughly corresponds to the --step option in RRDTool's graph/xport commands. Here is an explanation borrowed
	 * from RRDTool:<p>
	 * <p/>
	 * <i>"By default rrdgraph calculates the width of one pixel in the time
	 * domain and tries to get data at that resolution from the RRD. With
	 * this switch you can override this behavior. If you want rrdgraph to
	 * get data at 1 hour resolution from the RRD, then you can set the
	 * step to 3600 seconds. Note, that a step smaller than 1 pixel will
	 * be silently ignored."</i><p>
	 * <p/>
	 * I think this option is not that useful, but it's here just for compatibility.<p>
	 *
	 * @param step Time step at which data should be fetched from RRD files. If this method is not used,
	 *             the step will be equal to the smallest RRD step of all processed RRD files. If no RRD file is processed,
	 *             the step will be roughly equal to the with of one graph pixel (in seconds).
	 */
	public void setStep(long step) {
		this.step = step;
	}

	/**
	 * Returns the time step used for data processing. Initially, this method returns zero.
	 * Once {@link #processData()} is finished, the method will return the real value used for
	 * all internal computations. Roughly corresponds to the --step option in RRDTool's graph/xport commands.
	 *
	 * @return Step used for data processing.
	 */
	public long getStep() {
		return step;
	}

	/**
	 * Returns desired RRD archive step (reslution) in seconds to be used while fetching data
	 * from RRD files. In other words, this value will used as the last parameter of
	 * {@link RrdDb#createFetchRequest(String, long, long, long) RrdDb.createFetchRequest()} method
	 * when this method is called internally by this DataProcessor.
	 *
	 * @return Desired archive step (fetch resolution) in seconds.
	 */
	public long getFetchRequestResolution() {
		return fetchRequestResolution;
	}

	/**
	 * Sets desired RRD archive step in seconds to be used internally while fetching data
	 * from RRD files. In other words, this value will used as the last parameter of
	 * {@link RrdDb#createFetchRequest(String, long, long, long) RrdDb.createFetchRequest()} method
	 * when this method is called internally by this DataProcessor. If this method is never called, fetch
	 * request resolution defaults to 1 (smallest possible archive step will be chosen automatically).
	 *
	 * @param fetchRequestResolution Desired archive step (fetch resoltuion) in seconds.
	 */
	public void setFetchRequestResolution(long fetchRequestResolution) {
		this.fetchRequestResolution = fetchRequestResolution;
	}

	/**
	 * Returns ending timestamp. Basically, this value is equal to the ending timestamp
	 * specified in the constructor. However, if the ending timestamps was zero, it
	 * will be replaced with the real timestamp when the {@link #processData()} method returns. The real
	 * value will be calculated from the last update times of processed RRD files.
	 *
	 * @return Ending timestamp in seconds
	 */
	public long getEndingTimestamp() {
		return tEnd;
	}

	/**
	 * Returns consolidated timestamps created with the {@link #processData()} method.
	 *
	 * @return array of timestamps in seconds
	 * @throws RrdException thrown if timestamps are not calculated yet
	 */
	public long[] getTimestamps() throws RrdException {
		if (timestamps == null) {
			throw new RrdException("Timestamps not calculated yet");
		}
		else {
			return timestamps;
		}
	}

	/**
	 * Returns calculated values for a single datasource. Corresponding timestamps can be obtained from
	 * the {@link #getTimestamps()} method.
	 *
	 * @param sourceName Datasource name
	 * @return an array of datasource values
	 * @throws RrdException Thrown if invalid datasource name is specified,
	 *                      or if datasource values are not yet calculated (method {@link #processData()}
	 *                      was not called)
	 */
	public double[] getValues(String sourceName) throws RrdException {
		Source source = getSource(sourceName);
		double[] values = source.getValues();
		if (values == null) {
			throw new RrdException("Values not available for source [" + sourceName + "]");
		}
		return values;
	}

	/**
	 * Returns single aggregated value for a single datasource.
	 *
	 * @param sourceName Datasource name
	 * @param consolFun  Consolidation function to be applied to fetched datasource values.
	 *                   Valid consolidation functions are MIN, MAX, LAST, FIRST, AVERAGE and TOTAL
	 *                   (these string constants are conveniently defined in the {@link ConsolFuns} class)
	 * @return MIN, MAX, LAST, FIRST, AVERAGE or TOTAL value calculated from the data
	 *         for the given datasource name
	 * @throws RrdException Thrown if invalid datasource name is specified,
	 *                      or if datasource values are not yet calculated (method {@link #processData()}
	 *                      was not called)
	 */
	public double getAggregate(String sourceName, String consolFun) throws RrdException {
		Source source = getSource(sourceName);
		return source.getAggregates(tStart, tEnd).getAggregate(consolFun);
	}

	/**
	 * Returns all (MIN, MAX, LAST, FIRST, AVERAGE and TOTAL) aggregated values for a single datasource.
	 *
	 * @param sourceName Datasource name
	 * @return Object containing all aggregated values
	 * @throws RrdException Thrown if invalid datasource name is specified,
	 *                      or if datasource values are not yet calculated (method {@link #processData()}
	 *                      was not called)
	 */
	public Aggregates getAggregates(String sourceName) throws RrdException {
		Source source = getSource(sourceName);
		return source.getAggregates(tStart, tEnd);
	}

	/**
	 * This method is just an alias for {@link #getPercentile(String)} method.
	 * <p/>
	 * Used by ISPs which charge for bandwidth utilization on a "95th percentile" basis.<p>
	 * <p/>
	 * The 95th percentile is the highest source value left when the top 5% of a numerically sorted set
	 * of source data is discarded. It is used as a measure of the peak value used when one discounts
	 * a fair amount for transitory spikes. This makes it markedly different from the average.<p>
	 * <p/>
	 * Read more about this topic at
	 * <a href="http://www.red.net/support/resourcecentre/leasedline/percentile.php">Rednet</a> or
	 * <a href="http://www.bytemark.co.uk/support/tech/95thpercentile.html">Bytemark</a>.
	 *
	 * @param sourceName Datasource name
	 * @return 95th percentile of fetched source values
	 * @throws RrdException Thrown if invalid source name is supplied
	 */
	public double get95Percentile(String sourceName) throws RrdException {
		return getPercentile(sourceName);
	}

	/**
	 * Used by ISPs which charge for bandwidth utilization on a "95th percentile" basis.<p>
	 * <p/>
	 * The 95th percentile is the highest source value left when the top 5% of a numerically sorted set
	 * of source data is discarded. It is used as a measure of the peak value used when one discounts
	 * a fair amount for transitory spikes. This makes it markedly different from the average.<p>
	 * <p/>
	 * Read more about this topic at
	 * <a href="http://www.red.net/support/resourcecentre/leasedline/percentile.php">Rednet</a> or
	 * <a href="http://www.bytemark.co.uk/support/tech/95thpercentile.html">Bytemark</a>.
	 *
	 * @param sourceName Datasource name
	 * @return 95th percentile of fetched source values
	 * @throws RrdException Thrown if invalid source name is supplied
	 */
	public double getPercentile(String sourceName) throws RrdException {
		return getPercentile(sourceName, DEFAULT_PERCENTILE);
	}

	/**
	 * The same as {@link #getPercentile(String)} but with a possibility to define custom percentile boundary
	 * (different from 95).
	 *
	 * @param sourceName Datasource name.
	 * @param percentile Boundary percentile. Value of 95 (%) is suitable in most cases, but you are free
	 *                   to provide your own percentile boundary between zero and 100.
	 * @return Requested percentile of fetched source values
	 * @throws RrdException Thrown if invalid sourcename is supplied, or if the percentile value makes no sense.
	 */
	public double getPercentile(String sourceName, double percentile) throws RrdException {
		if (percentile <= 0.0 || percentile > 100.0) {
			throw new RrdException("Invalid percentile [" + percentile + "], should be between 0 and 100");
		}
		Source source = getSource(sourceName);
		return source.getPercentile(tStart, tEnd, percentile);
	}

	/**
	 * Returns array of datasource names defined in this DataProcessor.
	 *
	 * @return array of datasource names
	 */
	public String[] getSourceNames() {
		return sources.keySet().toArray(new String[0]);
	}

	/**
	 * Returns an array of all datasource values for all datasources. Each row in this two-dimensional
	 * array represents an array of calculated values for a single datasource. The order of rows is the same
	 * as the order in which datasources were added to this DataProcessor object.
	 *
	 * @return All datasource values for all datasources. The first index is the index of the datasource,
	 *         the second index is the index of the datasource value. The number of datasource values is equal
	 *         to the number of timestamps returned with {@link #getTimestamps()}  method.
	 * @throws RrdException Thrown if invalid datasource name is specified,
	 *                      or if datasource values are not yet calculated (method {@link #processData()}
	 *                      was not called)
	 */
	public double[][] getValues() throws RrdException {
		String[] names = getSourceNames();
		double[][] values = new double[names.length][];
		for (int i = 0; i < names.length; i++) {
			values[i] = getValues(names[i]);
		}
		return values;
	}

	private Source getSource(String sourceName) throws RrdException {
		Source source = sources.get(sourceName);
		if (source != null) {
			return source;
		}
		throw new RrdException("Unknown source: " + sourceName);
	}

	/////////////////////////////////////////////////////////////////
	// DATASOURCE DEFINITIONS
	/////////////////////////////////////////////////////////////////

	/**
	 * <p>Adds a custom, {@link org.jrobin.data.Plottable plottable} datasource (<b>PDEF</b>).
	 * The datapoints should be made available by a class extending
	 * {@link org.jrobin.data.Plottable Plottable} class.</p>
	 *
	 * @param name	  source name.
	 * @param plottable class that extends Plottable class and is suited for graphing.
	 */
	public void addDatasource(String name, Plottable plottable) {
		PDef pDef = new PDef(name, plottable);
		sources.put(name, pDef);
	}

	/**
	 * <p>Adds complex source (<b>CDEF</b>).
	 * Complex sources are evaluated using the supplied <code>RPN</code> expression.</p>
	 * <p/>
	 * <p>Complex source <code>name</code> can be used:</p>
	 * <ul>
	 * <li>To specify sources for line, area and stack plots.</li>
	 * <li>To define other complex sources.</li>
	 * </ul>
	 * <p/>
	 * <p>JRobin supports the following RPN functions, operators and constants: +, -, *, /,
	 * %, SIN, COS, LOG, EXP, FLOOR, CEIL, ROUND, POW, ABS, SQRT, RANDOM, LT, LE, GT, GE, EQ,
	 * IF, MIN, MAX, LIMIT, DUP, EXC, POP, UN, UNKN, NOW, TIME, PI, E,
	 * AND, OR, XOR, PREV, PREV(sourceName), INF, NEGINF, STEP, YEAR, MONTH, DATE,
	 * HOUR, MINUTE, SECOND, WEEK, SIGN and RND.</p>
	 * <p/>
	 * <p>JRobin does not force you to specify at least one simple source name as RRDTool.</p>
	 * <p/>
	 * <p>For more details on RPN see RRDTool's
	 * <a href="http://people.ee.ethz.ch/~oetiker/webtools/rrdtool/manual/rrdgraph.html" target="man">
	 * rrdgraph man page</a>.</p>
	 *
	 * @param name		  source name.
	 * @param rpnExpression RPN expression containig comma (or space) delimited simple and complex
	 *                      source names, RPN constants, functions and operators.
	 */
	public void addDatasource(String name, String rpnExpression) {
		CDef cDef = new CDef(name, rpnExpression);
		sources.put(name, cDef);
	}

	/**
	 * <p>Adds static source (<b>SDEF</b>). Static sources are the result of a consolidation function applied
	 * to *any* other source that has been defined previously.</p>
	 *
	 * @param name	  source name.
	 * @param defName   Name of the datasource to calculate the value from.
	 * @param consolFun Consolidation function to use for value calculation
	 */
	public void addDatasource(String name, String defName, String consolFun) {
		SDef sDef = new SDef(name, defName, consolFun);
		sources.put(name, sDef);
	}

	/**
	 * <p>Adds simple datasource (<b>DEF</b>). Simple source <code>name</code>
	 * can be used:</p>
	 * <ul>
	 * <li>To specify sources for line, area and stack plots.</li>
	 * <li>To define complex sources
	 * </ul>
	 *
	 * @param name	   source name.
	 * @param file	   Path to RRD file.
	 * @param dsName	 Datasource name defined in the RRD file.
	 * @param consolFunc Consolidation function that will be used to extract data from the RRD
	 *                   file ("AVERAGE", "MIN", "MAX" or "LAST" - these string constants are conveniently defined
	 *                   in the {@link org.jrobin.core.ConsolFuns ConsolFuns} class).
	 */
	public void addDatasource(String name, String file, String dsName, String consolFunc) {
		Def def = new Def(name, file, dsName, consolFunc);
		sources.put(name, def);
	}

	/**
	 * <p>Adds simple source (<b>DEF</b>). Source <code>name</code> can be used:</p>
	 * <ul>
	 * <li>To specify sources for line, area and stack plots.</li>
	 * <li>To define complex sources
	 * </ul>
	 *
	 * @param name	   Source name.
	 * @param file	   Path to RRD file.
	 * @param dsName	 Data source name defined in the RRD file.
	 * @param consolFunc Consolidation function that will be used to extract data from the RRD
	 *                   file ("AVERAGE", "MIN", "MAX" or "LAST" - these string constants are conveniently defined
	 *                   in the {@link org.jrobin.core.ConsolFuns ConsolFuns} class).
	 * @param backend	Name of the RrdBackendFactory that should be used for this RrdDb.
	 */
	public void addDatasource(String name, String file, String dsName, String consolFunc, String backend) {
		Def def = new Def(name, file, dsName, consolFunc, backend);
		sources.put(name, def);
	}

	/**
	 * Adds DEF datasource with datasource values already available in the FetchData object. This method is
	 * used internally by JRobin and probably has no purpose outside of it.
	 *
	 * @param name	  Source name.
	 * @param fetchData Fetched data containing values for the given source name.
	 */
	public void addDatasource(String name, FetchData fetchData) {
		Def def = new Def(name, fetchData);
		sources.put(name, def);
	}
	
	/**
         * Creates a new VDEF datasource that performs a percentile calculation on an
         * another named datasource to yield a single value.  
         * 
         * Requires that the other datasource has already been defined; otherwise, it'll
         * end up with no data
         * 
         * @param name - the new virtual datasource name
         * @param sourceName - the datasource from which to extract the percentile.  Must be a previously
         *                     defined virtual datasource
         * @param percentile - the percentile to extract from the source datasource
         */
	public void addDatasource(String name, String sourceName, double percentile) {
	    Source source = sources.get(sourceName);
	    sources.put(name, new PercentileDef(name, source, percentile));
	}

	/////////////////////////////////////////////////////////////////
	// CALCULATIONS
	/////////////////////////////////////////////////////////////////

	/**
	 * Method that should be called once all datasources are defined. Data will be fetched from
	 * RRD files, RPN expressions will be calculated, etc.
	 *
	 * @throws IOException  Thrown in case of I/O error (while fetching data from RRD files)
	 * @throws RrdException Thrown in case of JRobin specific error
	 */
	public void processData() throws IOException, RrdException {
		extractDefs();
		fetchRrdData();
		fixZeroEndingTimestamp();
		chooseOptimalStep();
		createTimestamps();
		assignTimestampsToSources();
		normalizeRrdValues();
		calculateNonRrdSources();
	}

	/**
	 * Method used to calculate datasource values which should be presented on the graph
	 * based on the desired graph width. Each value returned represents a single pixel on the graph.
	 * Corresponding timestamp can be found in the array returned from {@link #getTimestampsPerPixel()}
	 * method.
	 *
	 * @param sourceName Datasource name
	 * @param pixelCount Graph width
	 * @return Per-pixel datasource values
	 * @throws RrdException Thrown if datasource values are not yet calculated (method {@link #processData()}
	 *                      was not called)
	 */
	public double[] getValuesPerPixel(String sourceName, int pixelCount) throws RrdException {
		setPixelCount(pixelCount);
		return getValuesPerPixel(sourceName);
	}

	/**
	 * Method used to calculate datasource values which should be presented on the graph
	 * based on the graph width set with a {@link #setPixelCount(int)} method call.
	 * Each value returned represents a single pixel on the graph. Corresponding timestamp can be
	 * found in the array returned from {@link #getTimestampsPerPixel()} method.
	 *
	 * @param sourceName Datasource name
	 * @return Per-pixel datasource values
	 * @throws RrdException Thrown if datasource values are not yet calculated (method {@link #processData()}
	 *                      was not called)
	 */
	public double[] getValuesPerPixel(String sourceName) throws RrdException {
		double[] values = getValues(sourceName);
		double[] pixelValues = new double[pixelCount];
		Arrays.fill(pixelValues, Double.NaN);
		long span = tEnd - tStart;
		// this is the ugliest nested loop I have ever made
		for (int pix = 0, ref = 0; pix < pixelCount; pix++) {
			double t = tStart + (double) (span * pix) / (double) (pixelCount - 1);
			while (ref < timestamps.length) {
				if (t <= timestamps[ref] - step) {
					// too left, nothing to do, already NaN
					break;
				}
				else if (t <= timestamps[ref]) {
					// in brackets, get this value
					pixelValues[pix] = values[ref];
					break;
				}
				else {
					// too right
					ref++;
				}
			}
		}
		return pixelValues;
	}

	/**
	 * Calculates timestamps which correspond to individual pixels on the graph.
	 *
	 * @param pixelCount Graph width
	 * @return Array of timestamps
	 */
	public long[] getTimestampsPerPixel(int pixelCount) {
		setPixelCount(pixelCount);
		return getTimestampsPerPixel();
	}

	/**
	 * Calculates timestamps which correspond to individual pixels on the graph
	 * based on the graph width set with a {@link #setPixelCount(int)} method call.
	 *
	 * @return Array of timestamps
	 */
	public long[] getTimestampsPerPixel() {
		long[] times = new long[pixelCount];
		long span = tEnd - tStart;
		for (int i = 0; i < pixelCount; i++) {
			times[i] = Math.round(tStart + (double) (span * i) / (double) (pixelCount - 1));
		}
		return times;
	}

	/**
	 * Dumps timestamps and values of all datasources in a tabelar form. Very useful for debugging.
	 *
	 * @return Dumped object content.
	 * @throws RrdException Thrown if nothing is calculated so far (the method {@link #processData()}
	 *                      was not called).
	 */
	public String dump() throws RrdException {
		String[] names = getSourceNames();
		double[][] values = getValues();
		StringBuffer buffer = new StringBuffer();
		buffer.append(format("timestamp", 12));
		for (String name : names) {
			buffer.append(format(name, 20));
		}
		buffer.append("\n");
		for (int i = 0; i < timestamps.length; i++) {
			buffer.append(format("" + timestamps[i], 12));
			for (int j = 0; j < names.length; j++) {
				buffer.append(format(Util.formatDouble(values[j][i]), 20));
			}
			buffer.append("\n");
		}
		return buffer.toString();
	}

	/**
	 * Returns time when last RRD archive was updated (all RRD files are considered).
	 *
	 * @return Last archive update time for all RRD files in this DataProcessor
	 */
	public long getLastRrdArchiveUpdateTime() {
		return lastRrdArchiveUpdateTime;
	}

	// PRIVATE METHODS

	private void extractDefs() {
		List<Def> defList = new ArrayList<Def>();
		for (Source source : sources.values()) {
			if (source instanceof Def) {
				defList.add((Def) source);
			}
		}
		defSources = defList.toArray(new Def[defList.size()]);
	}

	private void fetchRrdData() throws IOException, RrdException {
		long tEndFixed = (tEnd == 0) ? Util.getTime() : tEnd;
		for (int i = 0; i < defSources.length; i++) {
			if (!defSources[i].isLoaded()) {
				// not fetched yet
				Set<String> dsNames = new HashSet<String>();
				dsNames.add(defSources[i].getDsName());
				// look for all other datasources with the same path and the same consolidation function
				for (int j = i + 1; j < defSources.length; j++) {
					if (defSources[i].isCompatibleWith(defSources[j])) {
						dsNames.add(defSources[j].getDsName());
					}
				}
				// now we have everything
				RrdDb rrd = null;
				try {
					rrd = getRrd(defSources[i]);
					lastRrdArchiveUpdateTime = Math.max(lastRrdArchiveUpdateTime, rrd.getLastArchiveUpdateTime());
					FetchRequest req = rrd.createFetchRequest(defSources[i].getConsolFun(),
							tStart, tEndFixed, fetchRequestResolution);
					req.setFilter(dsNames);
					FetchData data = req.fetchData();
					defSources[i].setFetchData(data);
					for (int j = i + 1; j < defSources.length; j++) {
						if (defSources[i].isCompatibleWith(defSources[j])) {
							defSources[j].setFetchData(data);
						}
					}
				}
				finally {
					if (rrd != null) {
						releaseRrd(rrd, defSources[i]);
					}
				}
			}
		}
	}

	private void fixZeroEndingTimestamp() throws RrdException {
		if (tEnd == 0) {
			if (defSources.length == 0) {
				throw new RrdException("Could not adjust zero ending timestamp, no DEF source provided");
			}
			tEnd = defSources[0].getArchiveEndTime();
			for (int i = 1; i < defSources.length; i++) {
				tEnd = Math.min(tEnd, defSources[i].getArchiveEndTime());
			}
			if (tEnd <= tStart) {
				throw new RrdException("Could not resolve zero ending timestamp.");
			}
		}
	}

	// Tricky and ugly. Should be redesigned some time in the future
	private void chooseOptimalStep() {
		long newStep = Long.MAX_VALUE;
		for (Def defSource : defSources) {
			long fetchStep = defSource.getFetchStep(), tryStep = fetchStep;
			if (step > 0) {
				tryStep = Math.min(newStep, (((step - 1) / fetchStep) + 1) * fetchStep);
			}
			newStep = Math.min(newStep, tryStep);
		}
		if (newStep != Long.MAX_VALUE) {
			// step resolved from a RRD file
			step = newStep;
		}
		else {
			// choose step based on the number of pixels (useful for plottable datasources)
			step = Math.max((tEnd - tStart) / pixelCount, 1);
		}
	}

	private void createTimestamps() {
		long t1 = Util.normalize(tStart, step);
		long t2 = Util.normalize(tEnd, step);
		if (t2 < tEnd) {
			t2 += step;
		}
		int count = (int) (((t2 - t1) / step) + 1);
		timestamps = new long[count];
		for (int i = 0; i < count; i++) {
			timestamps[i] = t1;
			t1 += step;
		}
	}

	private void assignTimestampsToSources() {
		for (Source src : sources.values()) {
			src.setTimestamps(timestamps);
		}
	}

	private void normalizeRrdValues() throws RrdException {
		Normalizer normalizer = new Normalizer(timestamps);
		for (Def def : defSources) {
			long[] rrdTimestamps = def.getRrdTimestamps();
			double[] rrdValues = def.getRrdValues();
			double[] values = normalizer.normalize(rrdTimestamps, rrdValues);
			def.setValues(values);
		}
	}

	private void calculateNonRrdSources() throws RrdException {
		for (Source source : sources.values()) {
			if (source instanceof SDef) {
				calculateSDef((SDef) source);
			}
			else if (source instanceof CDef) {
				calculateCDef((CDef) source);
			}
			else if (source instanceof PDef) {
				calculatePDef((PDef) source);
			} 
			else if (source instanceof PercentileDef) {
			        calculatePercentileDef((PercentileDef) source);
			}
		}
	}

	private void calculatePDef(PDef pdef) {
		pdef.calculateValues();
	}

	private void calculateCDef(CDef cDef) throws RrdException {
		RpnCalculator calc = new RpnCalculator(cDef.getRpnExpression(), cDef.getName(), this);
		cDef.setValues(calc.calculateValues());
	}

	private void calculateSDef(SDef sDef) throws RrdException {
		String defName = sDef.getDefName();
		String consolFun = sDef.getConsolFun();
		Source source = getSource(defName);
		double value = source.getAggregates(tStart, tEnd).getAggregate(consolFun);
		sDef.setValue(value);
	}
	
	//Yeah, this is different from the other calculation methods
	// Frankly, this is how it *should* be done, and the other methods will
	// be refactored to this design (and the instanceof's removed) at some point
        private void calculatePercentileDef(PercentileDef def) throws RrdException {
                def.calculate(tStart, tEnd);
        }


	private RrdDb getRrd(Def def) throws IOException, RrdException {
		String path = def.getPath(), backend = def.getBackend();
		if (poolUsed && backend == null) {
			return RrdDbPool.getInstance().requestRrdDb(path);
		}
		else if (backend != null) {
			return new RrdDb(path, true, RrdBackendFactory.getFactory(backend));
		}
		else {
			return new RrdDb(path, true);
		}
	}

	private void releaseRrd(RrdDb rrd, Def def) throws IOException, RrdException {
		String backend = def.getBackend();
		if (poolUsed && backend == null) {
			RrdDbPool.getInstance().release(rrd);
		}
		else {
			rrd.close();
		}
	}

	private static String format(String s, int length) {
		StringBuffer b = new StringBuffer(s);
		for (int i = 0; i < length - s.length(); i++) {
			b.append(' ');
		}
		return b.toString();
	}

	/**
	 * Cute little demo. Uses demo.rrd file previously created by basic JRobin demo.
	 *
	 * @param args Not used
	 * @throws IOException
	 * @throws RrdException
	 */
	public static void main(String[] args) throws IOException, RrdException {
		// time span
		long t1 = Util.getTimestamp(2003, 4, 1);
		long t2 = Util.getTimestamp(2003, 5, 1);
		System.out.println("t1 = " + t1);
		System.out.println("t2 = " + t2);

		// RRD file to use
		String rrdPath = Util.getJRobinDemoPath("demo.rrd");

		// constructor
		DataProcessor dp = new DataProcessor(t1, t2);

		// uncomment and run again
		//dp.setFetchRequestResolution(86400);

		// uncomment and run again
		//dp.setStep(86500);

		// datasource definitions
		dp.addDatasource("X", rrdPath, "sun", "AVERAGE");
		dp.addDatasource("Y", rrdPath, "shade", "AVERAGE");
		dp.addDatasource("Z", "X,Y,+,2,/");
		dp.addDatasource("DERIVE[Z]", "Z,PREV(Z),-,STEP,/");
		dp.addDatasource("TREND[Z]", "DERIVE[Z],SIGN");
		dp.addDatasource("AVG[Z]", "Z", "AVERAGE");
		dp.addDatasource("DELTA", "Z,AVG[Z],-");

		// action
		long laptime = System.currentTimeMillis();
		//dp.setStep(86400);
		dp.processData();
		System.out.println("Data processed in " + (System.currentTimeMillis() - laptime) + " milliseconds\n---");
		System.out.println(dp.dump());

		// aggregates
		System.out.println("\nAggregates for X");
		Aggregates agg = dp.getAggregates("X");
		System.out.println(agg.dump());
		System.out.println("\nAggregates for Y");
		agg = dp.getAggregates("Y");
		System.out.println(agg.dump());

		// 95-percentile
		System.out.println("\n95-percentile for X: " + Util.formatDouble(dp.get95Percentile("X")));
		System.out.println("95-percentile for Y: " + Util.formatDouble(dp.get95Percentile("Y")));

		// lastArchiveUpdateTime
		System.out.println("\nLast archive update time was: " + dp.getLastRrdArchiveUpdateTime());
	}
}

