package freenet.support;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;

import freenet.node.FSParseException;
import freenet.support.io.Closer;
import freenet.support.io.LineReader;
import freenet.support.io.Readers;

import static java.util.Collections.emptyMap;

/**
 * @author amphibian
 *
 * Simple FieldSet type thing, which uses the standard Java facilities. Should always be written as 
 * UTF-8. Simpler encoding than Properties.
 * 
 * All of the methods treat the data as a tree, where levels are indicated by ".". Hence e.g.:
 * DirectKey=Value
 * Subset.Key=Value
 * Subset.Subset.Key=Value
 * 
 * DETAILS:
 * <key>=<value> 
 * Key is split into a tree via "."'s.
 * Value is a string.
 * Conversion methods are provided for most key types, one notable issue is arrays of string, 
 * which are separated by ";".
 * 
 * ALTERNATE FORMAT:
 * <key>==<standard base64 encoded value>
 * The value is encoded. We will use this later on to prevent problems when transferring noderefs 
 * (line breaks, whitespace get changes when people paste stuff etc), and to allow e.g. newlines 
 * in strings. For now we only *read* such formats.
 */
public class SimpleFieldSet {

    private final Map<String, String> values;
    private Map<String, SimpleFieldSet> subsets;
    private String endMarker;
    private final boolean shortLived;
    private final boolean alwaysUseBase64;
    protected String[] header;

    public static final char MULTI_LEVEL_CHAR = '.';
    public static final char MULTI_VALUE_CHAR = ';';
    public static final char KEYVALUE_SEPARATOR_CHAR = '=';
    private static final String[] EMPTY_STRING_ARRAY = new String[0];

    public SimpleFieldSet(boolean shortLived) {
        this(shortLived, false);
    }
    
    /**
     * Create a SimpleFieldSet.
     * @param shortLived If false, strings will be interned to ensure that they use as
     * little memory as possible. Only set to true if the SFS will be short-lived or
     * small.
     * @param alwaysUseBase64 If true, the SFS can contain newlines etc in values, and we will
     * always use base64 for the values if they contain such invalid characters.
     */
    public SimpleFieldSet(boolean shortLived, boolean alwaysUseBase64) {
        values = new HashMap<String, String>();
       	subsets = null;
       	this.shortLived = shortLived;
       	this.alwaysUseBase64 = alwaysUseBase64;
    }

    public SimpleFieldSet(BufferedReader br, boolean allowMultiple, boolean shortLived) throws IOException {
        this(br, allowMultiple, shortLived, false, false);
    }
    
    /**
     * Construct a SimpleFieldSet from reading a BufferedReader.
     * @param br
     * @param allowMultiple If true, multiple lines with the same field name will be
     * combined; if false, the constructor will throw.
     * @param shortLived If false, strings will be interned to ensure that they use as
     * little memory as possible. Only set to true if the SFS will be short-lived or
     * small.
     * @throws IOException If the buffer could not be read, or if there was a formatting
     * problem.
     */
    public SimpleFieldSet(BufferedReader br, boolean allowMultiple, boolean shortLived, boolean allowBase64, boolean alwaysBase64) throws IOException {
        this(shortLived, alwaysBase64);
        read(Readers.fromBufferedReader(br), allowMultiple, allowBase64);
    }

    /** Copy constructor */
    public SimpleFieldSet(SimpleFieldSet sfs){
    	values = new HashMap<String, String>(sfs.values);
    	if(sfs.subsets != null)
    		subsets = new HashMap<String, SimpleFieldSet>(sfs.subsets);
    	this.shortLived = false; // it's been copied!
    	this.header = sfs.header;
    	this.endMarker = sfs.endMarker;
    	this.alwaysUseBase64 = sfs.alwaysUseBase64;
    }

    public SimpleFieldSet(LineReader lis, int maxLineLength, int lineBufferSize, boolean utf8OrIso88591, boolean allowMultiple, boolean shortLived) throws IOException {
    	this(lis, maxLineLength, lineBufferSize, utf8OrIso88591, allowMultiple, shortLived, false);
    }

    public SimpleFieldSet(LineReader lis, int maxLineLength, int lineBufferSize, boolean utf8OrIso88591, boolean allowMultiple, boolean shortLived, boolean allowBase64) throws IOException {
    	this(shortLived);
    	read(lis, maxLineLength, lineBufferSize, utf8OrIso88591, allowMultiple, allowBase64);
    }

    /**
     * Construct from a string.
     * String format:
     * blah=blah
     * blah=blah
     * End
     * @param shortLived If false, strings will be interned to ensure that they use as
     * little memory as possible. Only set to true if the SFS will be short-lived or
     * small.
     * @throws IOException if the string is too short or invalid.
     */
    public SimpleFieldSet(String content, boolean allowMultiple, boolean shortLived, boolean allowBase64) throws IOException {
    	this(shortLived);
        StringReader sr = new StringReader(content);
        BufferedReader br = new BufferedReader(sr);
	    read(Readers.fromBufferedReader(br), allowMultiple, allowBase64);
    }
    
    /**
     * Construct from a {@link String} array.
     * <p>
     * Similar to {@link #SimpleFieldSet(String, boolean, boolean)},
     * but each item of array represents a single line
     * </p>
     * @param content to be parsed 
     * @param allowMultiple If {@code true}, multiple lines with the same field name will be
     * combined; if {@code false}, the constructor will throw.
     * @param shortLived If {@code false}, strings will be interned to ensure that they use as
     * little memory as possible. Only set to {@code true} if the SFS will be short-lived or
     * small.
     * @throws IOException
     */
    public SimpleFieldSet(String[] content, boolean allowMultiple, boolean shortLived, boolean allowBase64) throws IOException {
    	this(shortLived);
    	read(Readers.fromStringArray(content), allowMultiple, allowBase64);
    }
    
    /**
     * @see #read(LineReader, int, int, boolean, boolean)
     */
	private void read(LineReader lr, boolean allowMultiple, boolean allowBase64) throws IOException {
		read(lr, Integer.MAX_VALUE, 0x100, true, allowMultiple, allowBase64);
	}
	
	/**
	 * Read from stream. Format:
	 *
	 * # Header1
	 * # Header2
	 * key0=val0
	 * key1=val1
	 * # comment
	 * key2=val2
	 * End
	 *
	 * (headers and comments are optional)
	 *
	 * @param utfOrIso88591 If true, read as UTF-8, otherwise read as ISO-8859-1.
	 */
	private void read(LineReader br, int maxLength, int bufferSize, boolean utfOrIso88591, boolean allowMultiple, boolean allowBase64) throws IOException {
		boolean firstLine = true;
		boolean headerSection = true;
		List<String> headers = new ArrayList<String>();

		while (true) {
			String line = br.readLine(maxLength, bufferSize, utfOrIso88591);
			if (line == null) {
				if (firstLine) throw new EOFException();
				Logger.error(this, "No end marker");
				break;
			}
			if (line.isEmpty()) continue; // ignore
			firstLine = false;

			char first = line.charAt(0);
			if (first == '#') {
				if (headerSection) {
					headers.add(line.substring(1).trim());
				}

			} else {
				if (headerSection) {
					if (!headers.isEmpty()) { this.header = headers.toArray(new String[headers.size()]); }
					headerSection = false;
				}

				int index = line.indexOf(KEYVALUE_SEPARATOR_CHAR);
				if(index >= 0) {
					// Mapping
					String before = line.substring(0, index).trim();
					String after = line.substring(index+1);
					if(!after.isEmpty() && after.charAt(0) == '=' && allowBase64) {
						try {
							after = after.substring(1);
							after = after.replaceAll("\\s", "");
							after = Base64.decodeUTF8(after);
						} catch (IllegalBase64Exception e) {
							throw new IOException("Unable to decode UTF8, = should not be allowed as first character of a value");
						}
					}
					if(!shortLived) after = after.intern();
					put(before, after, allowMultiple, false, true);
				} else {
					endMarker = line;
					break;
				}
			}
		}
	}

	/** Get a value for a key as a String. This may be a top level value, or we will traverse the 
	 * tree, so can be used for any key=value or subset.subset.key=value etc.
	 * @param key The key to look up.
	 * @return The String value corresponding to the given key, or null if there is no such 
	 * key=value pair.
	 */
    public synchronized String get(String key) {
   		int idx = key.indexOf(MULTI_LEVEL_CHAR);
   		if(idx == -1)
   			return values.get(key);
   		else if(idx == 0)
			return (subset("") == null) ? null : subset("").get(key.substring(1));
		else {
   			if(subsets == null) return null;
   			String before = key.substring(0, idx);
   			String after = key.substring(idx+1);
   			SimpleFieldSet fs = subsets.get(before);
   			if(fs == null) return null;
   			return fs.get(after);
   		}
    }

    public String[] getAll(String key) {
    	String k = get(key);
    	if(k == null) return null;
    	return split(k);
    }

    /** Get a list of String's from a single value, encoded in Base64. This is useful for storing
     * arbitrary String's that may contain illegal characters - the MULTI_VALUE_CHAR, newlines, etc.
     */
    public String[] getAllEncoded(String key) throws IllegalBase64Exception {
        String k = get(key);
        if(k == null) return null;
        String[] ret = split(k);
        for(int i=0;i<ret.length;i++) {
            ret[i] = Base64.decodeUTF8(ret[i]);
        }
        return ret;
    }

    /** Split a set of String's delimeted by MULTI_VALUE_CHAR, accepting empty strings at both ends.
     * E.g. ";blah;blah;blah;;" will give ["", "blah", "blah", "blah", "", ""].
     * Java 7 split() would give ["blah","blah","blah"].
     */
    public static String[] split(String string) {
    	if(string == null) return EMPTY_STRING_ARRAY;
    	// Java 7 version of String.split() trims the extra delimeters at each end.
    	int emptyAtStart = 0;
    	for(;emptyAtStart<string.length() && string.charAt(emptyAtStart) == MULTI_VALUE_CHAR;emptyAtStart++);
    	if(emptyAtStart == string.length()) {
    	    String[] ret = new String[string.length()];
    	    for(int i=0;i<ret.length;i++) ret[i] = "";
    	    return ret;
    	}
    	int emptyAtEnd = 0;
    	for(int i=string.length()-1; i>=0 && string.charAt(i) == MULTI_VALUE_CHAR;i--) emptyAtEnd++;
    	string = string.substring(emptyAtStart, string.length() - emptyAtEnd);
    	String[] split = string.split(String.valueOf(MULTI_VALUE_CHAR)); // slower???
    	if(emptyAtStart != 0 || emptyAtEnd != 0) {
    	    String[] ret = new String[emptyAtStart+split.length+emptyAtEnd];
    	    System.arraycopy(split, 0, ret, emptyAtStart, split.length);
    	    split = ret;
    	    for(int i=0;i<split.length;i++)
    	        if(split[i] == null) split[i] = "";
    	}
    	return split;
	}

    /** Combine a list of String's into a single String, separating them by the MULTI_VALUE_CHAR. */
    private static String unsplit(String[] strings) {
		if (strings.length == 0) return "";
    	StringBuilder sb = new StringBuilder();
    	for(String s: strings) {
    		sb.append(s);
    		assert(s.indexOf(MULTI_VALUE_CHAR) == -1);
			sb.append(MULTI_VALUE_CHAR);
    	}
		// assert(sb.length() > 0) -- always true as strings.length != 0
		// remove last MULTI_VALUE_CHAR
		sb.deleteCharAt(sb.length()-1);
    	return sb.toString();
    }

    /**
     * Put contents of a fieldset, overwrite old values.
     */
    public void putAllOverwrite(SimpleFieldSet fs) {
    	for(Map.Entry<String, String> entry: fs.values.entrySet()) {
    		values.put(entry.getKey(), entry.getValue()); // overwrite old
    	}
    	if(fs.subsets == null) return;
	if(subsets == null) subsets = new HashMap<String, SimpleFieldSet>();
    	for(Map.Entry<String, SimpleFieldSet> entry: fs.subsets.entrySet()) {
    		String key = entry.getKey();
    		SimpleFieldSet hisFS = entry.getValue();
    		SimpleFieldSet myFS = subsets.get(key);
    		if(myFS != null) {
    			myFS.putAllOverwrite(hisFS);
    		} else {
    			subsets.put(key, hisFS);
    		}
    	}
    }

    /**
     * Set a key to a value. If the value already exists, throw IllegalStateException.
     * @param key The key.
     * @param value The value.
     */
    public void putSingle(String key, String value) {
    	if(value == null) return;
    	if(!shortLived) value = value.intern();
    	if(!put(key, value, false, false, false))
    		throw new IllegalStateException("Value already exists: "+value+" but want to set "+key+" to "+value);
    }

    /**
     * Aggregating put. Set a key to a value, if the value already exists, append to it.
     * If you do not need this functionality please use putOverwrite for a minimal
     * performance gain.
     *
     * @param key The key.
     * @param value The value.
     */
    public void putAppend(String key, String value) {
    	if(value == null) return;
    	if(!shortLived) value = value.intern();
    	put(key, value, true, false, false);
    }

    /**
     * Set a key to a value, overwriting any existing value if present.
     * This function is a little bit faster than putAppend() because it does not
     * check whether the key already exists.
     *
     * @param key The key.
     * @param value The value.
     */
    public void putOverwrite(String key, String value) {
    	if(value == null) return;
    	if(!shortLived) value = value.intern();
    	put(key, value, false, true, false);
    }

    /**
     * Set a key to a value.
     * @param key The key.
     * @param value The value.
     * @param allowMultiple If true, if the key already exists then the value will be
     * appended to the existing value. If false, we return false to indicate that the
     * old value is unchanged.
     * @return True unless allowMultiple was false and there was a pre-existing value,
     * or value was null.
     */
	private synchronized boolean put(String key, String value, boolean allowMultiple, boolean overwrite, boolean fromRead) {
		int idx;
		if(value == null) return true; // valid no-op
		if((!alwaysUseBase64) && value.indexOf('\n') != -1) throw new IllegalArgumentException("A simplefieldSet can't accept newlines !");
		if(allowMultiple && (!fromRead) && value.indexOf(MULTI_VALUE_CHAR) != -1) {
			throw new IllegalArgumentException("Appending a string to a SimpleFieldSet value should not contain the multi-value char \""+String.valueOf(MULTI_VALUE_CHAR)+"\" but it does: \"" +value+"\" for \""+key+"\"", new Exception("error"));
		}
		if((idx = key.indexOf(MULTI_LEVEL_CHAR)) == -1) {
			if(!shortLived) key = key.intern();

			if(overwrite) {
				values.put(key, value);
			} else {
				if(values.get(key) == null) {
					values.put(key, value);
				} else {
					if(!allowMultiple) return false;
					values.put(key, (values.get(key))+ MULTI_VALUE_CHAR +value);
				}
			}
		} else {
			String before = key.substring(0, idx);
			String after = key.substring(idx+1);
			SimpleFieldSet fs = null;
			if(subsets == null)
				subsets = new HashMap<String, SimpleFieldSet>();
			fs = subsets.get(before);
			if(fs == null) {
				fs = new SimpleFieldSet(shortLived, alwaysUseBase64);
				if(!shortLived) before = before.intern();
				subsets.put(before, fs);
			}
			fs.put(after, value, allowMultiple, overwrite, fromRead);
		}
		return true;
    }

	public void put(String key, int value) {
		// Use putSingle so it does the intern check
		putSingle(key, Integer.toString(value));
	}

	public void put(String key, long value) {
		putSingle(key, Long.toString(value));
	}

	public void put(String key, short value) {
		putSingle(key, Short.toString(value));
	}

	public void put(String key, char c) {
		putSingle(key, Character.toString(c));
	}

	public void put(String key, boolean b) {
		// Don't use putSingle, avoid intern check (Boolean.toString returns interned strings anyway)
		put(key, Boolean.toString(b), false, false, false);
	}

	public void put(String key, double windowSize) {
		putSingle(key, Double.toString(windowSize));
	}
	
	public void put(String key, byte[] bytes) {
	    putSingle(key, Base64.encode(bytes));
	}

    /**
     * Write the contents of the SimpleFieldSet to a Writer.
     * Note: The caller *must* buffer the writer to avoid lousy performance!
     * (StringWriter is by definition buffered, otherwise wrap it in a BufferedWriter)
     *
     * @warning keep in mind that a Writer is not necessarily UTF-8!!
     */
	public void writeTo(Writer w) throws IOException {
		writeTo(w, "", false, false);
	}

    /**
     * Write the contents of the SimpleFieldSet to a Writer.
     * Note: The caller *must* buffer the writer to avoid lousy performance!
     * (StringWriter is by definition buffered, otherwise wrap it in a BufferedWriter)
     * 
     * @param w The Writer to write to. @warning keep in mind that a Writer is not necessarily UTF-8!!
     * @param prefix String to prefix the keys with. (E.g. when writing a tree of SFS's).
     * @param noEndMarker If true, don't write the end marker (the last line, the only one with no
     * "=" in it).
     * @param useBase64 If true, use Base64 for any value that has control characters, whitespace, 
     * or characters used by SimpleFieldSet in it. In this case the separator will be "==" not "=".
     * This is mainly useful for node references, which tend to lose whitespace, gain newlines etc
     * in transit. Can be overridden (to true) by alwaysUseBase64 setting.
     */
    synchronized void writeTo(Writer w, String prefix, boolean noEndMarker, boolean useBase64) throws IOException {
		writeHeader(w);
    	for (Map.Entry<String, String> entry: values.entrySet()) {
			String key = entry.getKey();
			String value = entry.getValue();
			writeValue(w, key, value, prefix, useBase64);
    	}
    	if(subsets != null) {
    		for (Map.Entry<String, SimpleFieldSet> entry: subsets.entrySet()) {
				String key = entry.getKey();
				SimpleFieldSet subset = entry.getValue();
    			if(subset == null) throw new NullPointerException();
    			subset.writeTo(w, prefix+key+MULTI_LEVEL_CHAR, true, useBase64);
    		}
    	}
    	if(!noEndMarker) {
    		if(endMarker == null)
    			w.write("End\n");
    		else {
    			w.write(endMarker);
    			w.write('\n');
    		}
    	}
    }

    private void writeValue(Writer w, String key, String value, String prefix, boolean useBase64) throws IOException {
        w.write(prefix);
        w.write(key);
        w.write(KEYVALUE_SEPARATOR_CHAR);
        if((useBase64 || alwaysUseBase64) && shouldBase64(value)) {
        	w.write(KEYVALUE_SEPARATOR_CHAR);
        	w.write(Base64.encodeUTF8(value));
        } else {
        	w.write(value);
        }
        w.write('\n');
	}

	private boolean shouldBase64(String value) {
    	for(int i=0;i<value.length();i++) {
    		char c = value.charAt(i);
    		if(c == SimpleFieldSet.KEYVALUE_SEPARATOR_CHAR) return true;
    		if(c == SimpleFieldSet.MULTI_LEVEL_CHAR) return true;
    		if(c == SimpleFieldSet.MULTI_VALUE_CHAR) return true;
    		if(Character.isISOControl(c)) return true;
    		if(Character.isWhitespace(c)) return true;
    	}
    	return false;
	}

	public void writeToOrdered(Writer w) throws IOException {
		writeToOrdered(w, "", false, false);
	}

	/**
	 * Write the SimpleFieldSet to a Writer, in order.
	 * @param w The Writer to write the SFS to.
	 * @param prefix The prefix ("" at the top level, e.g. "something." if we are inside a sub-SFS 
	 * called "something").
	 * @param noEndMarker If true, don't write the end marker (usually End). Again this is normally
	 * set only in sub-SFS's.
	 * @param allowOptionalBase64 If true, write fields as Base64 if they contain spaces etc. This
	 * improves the robustness of e.g. node references, where the SFS can be written with or 
	 * without Base64. However, for SFS's where the values can contain <b>anything</b>, the member
	 * flag alwaysUseBase64 will be set and we will write lines that need to be Base64 as such 
	 * regardless of this allowOptionalBase64.
	 * @throws IOException If an error occurs writing to the Writer.
	 */
    private synchronized void writeToOrdered(Writer w, String prefix, boolean noEndMarker, boolean allowOptionalBase64) throws IOException {
		writeHeader(w);
    	String[] keys = values.keySet().toArray(new String[values.size()]);
    	int i=0;

    	// Sort
    	Arrays.sort(keys);

    	// Output
    	for(i=0; i < keys.length; i++) {
    		writeValue(w, keys[i], get(keys[i]), prefix, allowOptionalBase64);
    	}

    	if(subsets != null) {
    		String[] orderedPrefixes = subsets.keySet().toArray(new String[subsets.size()]);
    		// Sort
    		Arrays.sort(orderedPrefixes);

        	for(i=0; i < orderedPrefixes.length; i++) {
    			SimpleFieldSet subset = subset(orderedPrefixes[i]);
    			if(subset == null) throw new NullPointerException();
    			subset.writeToOrdered(w, prefix+orderedPrefixes[i]+MULTI_LEVEL_CHAR, true, allowOptionalBase64);
    		}
    	}

    	if(!noEndMarker) {
    		if(endMarker == null)
    			w.write("End\n");
    		else
    			w.write(endMarker+ '\n');
    	}
    }

	private void writeHeader(Writer w) throws IOException {
		if (header != null) {
			for (String line: header) {
				w.write("# " + line + "\n");
			}
		}
	}

	@Override
    public String toString() {
        StringWriter sw = new StringWriter();
        try {
            writeTo(sw);
        } catch (IOException e) {
            Logger.error(this, "WTF?!: "+e+" in toString()!", e);
        }
        return sw.toString();
    }

    public String toOrderedString() {
    	StringWriter sw = new StringWriter();
        try {
            writeToOrdered(sw);
        } catch (IOException e) {
            Logger.error(this, "WTF?!: "+e+" in toString()!", e);
        }
        return sw.toString();
    }

    public String toOrderedStringWithBase64() {
        StringWriter sw = new StringWriter();
        try {
            writeToOrdered(sw, "", false, true);
        } catch (IOException e) {
            Logger.error(this, "WTF?!: "+e+" in toString()!", e);
        }
        return sw.toString();
    }

    public String getEndMarker() {
    	return endMarker;
    }

    public void setEndMarker(String s) {
    	endMarker = s;
    }

	public synchronized SimpleFieldSet subset(String key) {
		if(subsets == null) return null;
		int idx = key.indexOf(MULTI_LEVEL_CHAR);
		if(idx == -1)
			return subsets.get(key);
		String before = key.substring(0, idx);
		String after = key.substring(idx+1);
		SimpleFieldSet fs = subsets.get(before);
		if(fs == null) return null;
		return fs.subset(after);
	}

	/**
	 * Like subset(), only throws instead of returning null.
	 * @throws FSParseException
	 */
	public synchronized SimpleFieldSet getSubset(String key) throws FSParseException {
		SimpleFieldSet fs = subset(key);
		if(fs == null) throw new FSParseException("No such subset "+key);
		return fs;
	}

	/** Iterate over all keys in the SimpleFieldSet, even if they are at lower levels. */
	public Iterator<String> keyIterator() {
		return new KeyIterator("");
	}

	/** Iterate over all keys in the SimpleFieldSet, even if they are at lower levels. 
	 * @param prefix Add the given prefix to lower levels. This is used recursively by KeyIterator.
	 */
	public KeyIterator keyIterator(String prefix) {
		return new KeyIterator(prefix);
	}

	/** Iterate over keys that are in the top level of the tree, i.e. that do not contain a ".". 
	 * E.g. "Name=Value" is a top level key. "Subset.Name=Value" is NOT a top level key. */
	public Iterator<String> toplevelKeyIterator() {
	    return values.keySet().iterator();
	}
	
    public class KeyIterator implements Iterator<String> {
    	final Iterator<String> valuesIterator;
    	final Iterator<String> subsetIterator;
    	KeyIterator subIterator;
    	String prefix;

    	/**
    	 * It provides an iterator for the SimpleSetField
    	 * which passes through every key.
    	 * (e.g. for key1=value1 key2.sub2=value2 key1.sub=value3
    	 * it will provide key1,key2.sub2,key1.sub)
    	 * @param a prefix to put BEFORE every key
    	 * (e.g. for key1=value, if the iterator is created with prefix "aPrefix",
    	 * it will provide aPrefixkey1
    	 */
    	public KeyIterator(String prefix) {
    		synchronized(SimpleFieldSet.this) {
    			valuesIterator = values.keySet().iterator();
    			if(subsets != null)
    				subsetIterator = subsets.keySet().iterator();
    			else
    				subsetIterator = null;
    			while(true) {
    				if(valuesIterator != null && valuesIterator.hasNext()) break;
    				if(subsetIterator == null || !subsetIterator.hasNext()) break;
    				String name = subsetIterator.next();
    				if(name == null) continue;
    				SimpleFieldSet fs = subsets.get(name);
    				if(fs == null) continue;
    				String newPrefix = prefix + name + MULTI_LEVEL_CHAR;
    				subIterator = fs.keyIterator(newPrefix);
    				if(subIterator.hasNext()) break;
    				subIterator = null;
    			}
    			this.prefix = prefix;
    		}
    	}

		@Override
		public boolean hasNext() {
			synchronized(SimpleFieldSet.this) {
				while(true) {
					if(valuesIterator.hasNext()) return true;
					if((subIterator != null) && subIterator.hasNext()) return true;
					if(subIterator != null) subIterator = null;
					if(subsetIterator != null && subsetIterator.hasNext()) {
						String key = subsetIterator.next();
						SimpleFieldSet fs = subsets.get(key);
						String newPrefix = prefix + key + MULTI_LEVEL_CHAR;
						subIterator = fs.keyIterator(newPrefix);
					} else
						return false;
				}
			}
		}

		@Override
		public final String next() {
			return nextKey();
		}

		public String nextKey() {
			synchronized(SimpleFieldSet.this) {
				String ret = null;
				if(valuesIterator != null && valuesIterator.hasNext()) {
					return prefix + valuesIterator.next();
				}
				// Iterate subsets.
				while(true) {
					if(subIterator != null && subIterator.hasNext()) {
						// If we have a retval, and we have a next value, return
						if(ret != null) return ret;
						ret = subIterator.next();
						if(subIterator.hasNext())
							// If we have a retval, and we have a next value, return
							if(ret != null) return ret;
					}
					// Otherwise, we need to get a new subIterator (or hasNext() will return false)
					subIterator = null;
					if(subsetIterator != null && subsetIterator.hasNext()) {
						String key = subsetIterator.next();
						SimpleFieldSet fs = subsets.get(key);
						String newPrefix = prefix + key + MULTI_LEVEL_CHAR;
						subIterator = fs.keyIterator(newPrefix);
					} else {
						// No more subIterator's
						if(ret == null) {
							//There is nothing to return and no more iterators, so we must be out
							//of elements
							throw new NoSuchElementException();
						}
						return ret;
					}
				}
			}
		}

		@Override
		public synchronized void remove() {
			throw new UnsupportedOperationException();
		}
	}
    
    /** Get a read-only map of direct key name:value pairs. Direct key values are things like 
     * "Name=Value" (which would return a map containing "Name" -> "Value", NOT 
     * "Subset.Name=Value" (which would not be returned). */
    public Map<String, String> directKeyValues() {
        return Collections.unmodifiableMap(values);
    }

    /** Get a read-only set of direct key names. So:
     * Name=Value
     * Subset.OtherName=Value
     * End
     * Would give "Name".
     * @return
     */
    public Set<String> directKeys() {
        return Collections.unmodifiableSet(values.keySet());
    }

    /** Get a read-only set of direct subsets. So:
     * Name=Value
     * Subset.OtherName=Value
     * End
     * Would give "OtherName" -> SFS containing OtherName=Value.
     * @return
     */
    public Map<String, SimpleFieldSet> directSubsets() {
        return subsets == null ? emptyMap() : Collections.unmodifiableMap(subsets);
    }

    /** Tolerant put(); does nothing if fs is empty */
    public void tput(String key, SimpleFieldSet fs) {
    	if(fs == null || fs.isEmpty()) return;
    	put(key, fs);
    }

    /** Add a name:value pair, traversing the tree and creating sub-SFS's if necessary. So we can
     * add("a.b.c.d", "value) even if there is no subset "a"; it will create it automatically.
     * @param key Name of the key to add.
     * @param fs Subset under the key.
     */
	public void put(String key, SimpleFieldSet fs) {
		if(fs == null) return; // legal no-op, because used everywhere
		if(fs.isEmpty()) // can't just no-op, because caller might add the FS then populate it...
			throw new IllegalArgumentException("Empty");
		if(subsets == null)
			subsets = new HashMap<String, SimpleFieldSet>();
		if(subsets.containsKey(key))
			throw new IllegalArgumentException("Already contains "+key+" but trying to add a SimpleFieldSet!");
		if(!shortLived) key = key.intern();
		subsets.put(key, fs);
	}

	/** Remove a name:value pair at any point in the tree. Will automatically traverse the tree and 
	 * remove empty subsets (which are not written anyway). */
	public synchronized void removeValue(String key) {
		int idx;
		if((idx = key.indexOf(MULTI_LEVEL_CHAR)) == -1) {
			values.remove(key);
		} else {
			if(subsets == null) return;
			String before = key.substring(0, idx);
			String after = key.substring(idx+1);
			SimpleFieldSet fs = subsets.get(before);
			if(fs == null) {
				return;
			}
			fs.removeValue(after);
			if(fs.isEmpty()) {
				subsets.remove(before);
				if(subsets.isEmpty())
					subsets = null;
			}
		}
	}

	/**
	 * It removes the specified subset.
	 * For example, in a SimpleFieldSet like this:
	 * foo=bar
	 * foo.bar=foobar
	 * foo.bar.boo=foobarboo
	 * calling it with the parameter "foo"
	 * means to drop the second and the third line.
	 * @param is the subset to remove
	 */
	public synchronized void removeSubset(String key) {
		if(subsets == null) return;
		int idx;
		if((idx = key.indexOf(MULTI_LEVEL_CHAR)) == -1) {
			subsets.remove(key);
		} else {
			String before = key.substring(0, idx);
			String after = key.substring(idx+1);
			SimpleFieldSet fs = subsets.get(before);
			if(fs == null) {
				return;
			}
			fs.removeSubset(after);
			if(fs.isEmpty()) {
				subsets.remove(before);
				if(subsets.isEmpty())
					subsets = null;
			}
		}
	}

	/** Is this SimpleFieldSet empty? */
	public synchronized boolean isEmpty() {
		return values.isEmpty() && (subsets == null || subsets.isEmpty());
	}

	/** Iterator over the names of direct subsets, i.e. the tree nodes just below this one, not the
	 * values. E.g.:
	 * Foo.Bar.Bat=1
	 * Baz.Boo=hello
	 * Grrr=goodbye
	 * End
	 * Returns "Foo", "Baz".
	 */
	public Iterator<String> directSubsetNameIterator() {
		return (subsets == null) ? null : subsets.keySet().iterator();
	}

	/** Get the names of direct subsets, i.e. the tree nodes just below this one, not the
     * values. E.g.:
     * Foo.Bar.Bat=1
     * Baz.Boo=hello
     * Grrr=goodbye
     * End
     * Returns [ "Foo", "Baz" ].
     */
	public String[] namesOfDirectSubsets() {
		return (subsets == null) ? EMPTY_STRING_ARRAY : subsets.keySet().toArray(new String[subsets.size()]);
	}

	/** Read a SimpleFieldSet from an InputStream in the standard format, using UTF-8, and not 
	 * allowing the Base64 encoding.
	 * @param is The InputStream to read from. We will use the UTF-8 charset.
     * @param allowMultiple Whether to allow multiple entries for each key (and automatically 
     * combine them). Not usually useful except maybe in FCP.
     * @param shortLived If true, don't intern the strings.
     * @return A new SimpleFieldSet.
     * @throws IOException If a read error occurs, including a formatting error, illegal 
     * characters etc.
	 */
	public static SimpleFieldSet readFrom(InputStream is, boolean allowMultiple, boolean shortLived) throws IOException {
	    return readFrom(is, allowMultiple, shortLived, false, false);
	}

	/**
	 * Read a SimpleFieldSet from an InputStream.
	 * @param is The InputStream to read from. We will use the UTF-8 charset.
	 * @param allowMultiple Whether to allow multiple entries for each key (and automatically 
	 * combine them). Not usually useful except maybe in FCP.
	 * @param shortLived If true, don't intern the strings.
	 * @param allowBase64 If true, allow reading Base64 encoded lines (key==base64(value)).
	 * @param alwaysBase64 If true, the resulting SFS should have the alwaysUseBase64 flag enabled, 
	 * i.e it can store anything in key values including newlines, special chars such as = etc. 
	 * Otherwise, even if allowBase64 is enabled, invalid chars will not be.
	 * @return A new SimpleFieldSet.
	 * @throws IOException If a read error occurs, including a formatting error, illegal 
	 * characters etc.
	 */
	public static SimpleFieldSet readFrom(InputStream is, boolean allowMultiple, boolean shortLived, boolean allowBase64, boolean alwaysBase64) throws IOException {
		BufferedInputStream bis = null;
		InputStreamReader isr = null;
		BufferedReader br = null;

		try {
			bis = new BufferedInputStream(is);
			isr = new InputStreamReader(bis, StandardCharsets.UTF_8);
			br = new BufferedReader(isr);
			SimpleFieldSet fs = new SimpleFieldSet(br, allowMultiple, shortLived, allowBase64, alwaysBase64);
			br.close();

			return fs;
		} finally {
                        Closer.close(br);
                        Closer.close(isr);
                        Closer.close(bis);
                }
	}

	/** Read a SimpleFieldSet from a File. */
	public static SimpleFieldSet readFrom(File f, boolean allowMultiple, boolean shortLived) throws IOException {
	    FileInputStream fis = new FileInputStream(f);
	    try {
	        return readFrom(fis, allowMultiple, shortLived);
	    } finally {
	        fis.close();
	    }
	}

    /** Write to the given OutputStream (as UTF-8) and flush it. */
    public void writeTo(OutputStream os) throws IOException {
        writeTo(os, 4096);
    }   
    
	/** Write to the given OutputStream and flush it. Use a big buffer, for jobs that aren't called
	 * too often e.g. persisting a file every 10 minutes. */
    public void writeToBigBuffer(OutputStream os) throws IOException {
    	writeTo(os, 65536);
    }	
	
	/** Write to the given OutputStream and flush it. */
    public void writeTo(OutputStream os, int bufferSize) throws IOException {
        BufferedOutputStream bos = null;
        OutputStreamWriter osw = null;
        BufferedWriter bw = null;
        
        bos = new BufferedOutputStream(os, bufferSize);
        osw = new OutputStreamWriter(bos, StandardCharsets.UTF_8);
        bw = new BufferedWriter(osw);
        writeTo(bw);
        bw.flush();
    }

    /** Get an integer value for the given key. This may be at the top level or lower in the tree,
     * it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @param def The default value to return if the key does not exist or can't be parsed.
     * @return The integer value of the key, or the default value.
     */
	public int getInt(String key, int def) {
		String s = get(key);
		if(s == null) return def;
		try {
			return Integer.parseInt(s);
		} catch (NumberFormatException e) {
			return def;
		}
	}

    /** Get an integer value for the given key. This may be at the top level or lower in the tree,
     * it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The integer value of the key, if it exists and is valid.
     * @throws FSParseException If the key=value pair does not exist or if the value cannot be 
     * parsed as an integer.
     */
	public int getInt(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No key "+key);
		try {
			return Integer.parseInt(s);
		} catch (NumberFormatException e) {
			throw new FSParseException("Cannot parse "+s+" for integer "+key);
		}
	}

    /** Get a double precision value for the given key. This may be at the top level or lower in 
     * the tree, it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @param def The default value to return if the key does not exist or can't be parsed.
     * @return The integer value of the key, or the default value.
     */
	public double getDouble(String key, double def) {
		String s = get(key);
		if(s == null) return def;
		try {
			return Double.parseDouble(s);
		} catch (NumberFormatException e) {
			return def;
		}
	}

    /** Get a double precision value for the given key. This may be at the top level or lower in 
     * the tree, it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The value of the key as a double, if it exists and is valid.
     * @throws FSParseException If the key=value pair does not exist or if the value cannot be 
     * parsed as a double.
     */
	public double getDouble(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No key "+key);
		try {
			return Double.parseDouble(s);
		} catch (NumberFormatException e) {
			throw new FSParseException("Cannot parse "+s+" for integer "+key);
		}
	}

    /** Get a long value for the given key. This may be at the top level or lower in the tree, 
     * it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @param def The default value to return if the key does not exist or can't be parsed.
     * @return The long value of the key, or the default value.
     */
	public long getLong(String key, long def) {
		String s = get(key);
		if(s == null) return def;
		try {
			return Long.parseLong(s);
		} catch (NumberFormatException e) {
			return def;
		}
	}

    /** Get a long value for the given key. This may be at the top level or lower in the tree, 
     * it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The value of the key as a long, if it exists and is valid.
     * @throws FSParseException If the key=value pair does not exist or if the value cannot be 
     * parsed as a long.
     */
	public long getLong(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No key "+key);
		try {
			return Long.parseLong(s);
		} catch (NumberFormatException e) {
			throw new FSParseException("Cannot parse "+s+" for long "+key);
		}
	}

    /** Get a short value for the given key. This may be at the top level or lower in the tree, 
     * it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The value of the key as a short, if it exists and is valid.
     * @throws FSParseException If the key=value pair does not exist or if the value cannot be 
     * parsed as a short.
     */
	public short getShort(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No key "+key);
		try {
			return Short.parseShort(s);
		} catch (NumberFormatException e) {
			throw new FSParseException("Cannot parse "+s+" for short "+key);
		}
	}

    /** Get a short value for the given key. This may be at the top level or lower in the tree, 
     * it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The value of the key as a short, if it exists and is valid.
     */
	public short getShort(String key, short def) {
		String s = get(key);
		if(s == null) return def;
		try {
			return Short.parseShort(s);
		} catch (NumberFormatException e) {
			return def;
		}
	}

	/** Get a byte value for the given key (represented as a number in decimal). This may be at 
	 * the top level or lower in the tree, it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The value of the key as a byte, if it exists and is valid.
     * @throws FSParseException If the key=value pair does not exist or if the value cannot be 
     * parsed as a byte.
     */
	public byte getByte(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No key " + key);
		try {
			return Byte.parseByte(s);
		} catch (NumberFormatException e) {
			throw new FSParseException("Cannot parse \"" + s + "\" as a byte.");
		}
	}

    /** Get a byte value for the given key (represented as a number in decimal). This may be at 
     * the top level or lower in the tree, it's just key=value. (Value in decimal)
     * @param key The key to fetch.
     * @return The value of the key as a byte, if it exists and is valid, otherwise the default
     * value.
     */
	public byte getByte(String key, byte def) {
		try {
			return getByte(key);
		} catch (FSParseException e) {
			return def;
		}
	}

	/** Get a byte array for the given key (represented in Base64). The key may be at the top level
	 * or further down the tree, so this is key=[base64 of value].
	 * @param key The key to fetch.
	 * @return The byte array to fetch.
	 * @throws FSParseException If the key does not exist or cannot be parsed as a byte array.
	 */
	public byte[] getByteArray(String key) throws FSParseException {
        String s = get(key);
        if(s == null) throw new FSParseException("No key " + key);
        try {
            return Base64.decode(s);
        } catch (IllegalBase64Exception e) {
            throw new FSParseException("Cannot parse value \""+s+"\" as a byte[]");
        }
	}

	/** Get a char for the given key (represented as a single character). The key may be at the 
	 * top level or further down the tree, so this is key=[character].
     * @param key The key to fetch.
	 * @return The character to fetch.
	 * @throws FSParseException If the key does not exist or there is more than one character.
	 */
	public char getChar(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No key "+key);
			if (s.length() == 1)
				return s.charAt(0);
			else
				throw new FSParseException("Cannot parse "+s+" for char "+key);
	}

    /** Get a char for the given key (represented as a single character). The key may be at the 
     * top level or further down the tree, so this is key=[character].
     * @param key The key to fetch.
     * @param def The default value to return if the key does not exist or can't be parsed.
     * @return The character to fetch.
     * @throws FSParseException If the key does not exist or there is more than one character.
     */
	public char getChar(String key, char def) {
		String s = get(key);
		if(s == null) return def;
			if (s.length() == 1)
				return s.charAt(0);
			else
				return def;
	}

	public boolean getBoolean(String key, boolean def) {
		return Fields.stringToBool(get(key), def);
	}

	public boolean getBoolean(String key) throws FSParseException {
		try {
		    return Fields.stringToBool(get(key));
		} catch(NumberFormatException e) {
		    throw new FSParseException(e);
		}
	}

	public void put(String key, int[] value) {
		removeValue(key);
		for(int v : value)
			putAppend(key, String.valueOf(v));
	}

	public void put(String key, double[] value) {
		removeValue(key);
		for(double v : value)
			putAppend(key, String.valueOf(v));
	}

	public void put(String key, float[] value) {
		removeValue(key);
		for (float v : value) putAppend(key, String.valueOf(v));
	}
	
    public void put(String key, short[] value) {
        removeValue(key);
        for (short v : value) putAppend(key, String.valueOf(v));
    }
    
    public void put(String key, long[] value) {
        removeValue(key);
        for (long v : value) putAppend(key, String.valueOf(v));
    }
    
    public void put(String key, boolean[] value) {
        removeValue(key);
        for (boolean v : value) putAppend(key, String.valueOf(v));
    }
    
	public int[] getIntArray(String key) {
		String[] strings = getAll(key);
		if(strings == null) return null;
		int[] ret = new int[strings.length];
		for(int i=0;i<strings.length;i++) {
			try {
				ret[i] = Integer.parseInt(strings[i]);
			} catch (NumberFormatException e) {
				Logger.error(this, "Cannot parse "+strings[i]+" : "+e, e);
				return null;
			}
		}
		return ret;
	}

    public short[] getShortArray(String key) {
        String[] strings = getAll(key);
        if(strings == null) return null;
        short[] ret = new short[strings.length];
        for(int i=0;i<strings.length;i++) {
            try {
                ret[i] = Short.parseShort(strings[i]);
            } catch (NumberFormatException e) {
                Logger.error(this, "Cannot parse "+strings[i]+" : "+e, e);
                return null;
            }
        }
        return ret;
    }

    public long[] getLongArray(String key) {
        String[] strings = getAll(key);
        if(strings == null) return null;
        long[] ret = new long[strings.length];
        for(int i=0;i<strings.length;i++) {
            try {
                ret[i] = Long.parseLong(strings[i]);
            } catch (NumberFormatException e) {
                Logger.error(this, "Cannot parse "+strings[i]+" : "+e, e);
                return null;
            }
        }
        return ret;
    }

	public double[] getDoubleArray(String key) {
		String[] strings = getAll(key);
		if(strings == null) return null;
		double[] ret = new double[strings.length];
		for(int i=0;i<strings.length;i++) {
			try {
				ret[i] = Double.parseDouble(strings[i]);
			} catch(NumberFormatException e) {
				Logger.error(this, "Cannot parse "+strings[i]+" : "+e,e);
				return null;
			}
		}

		return ret;
	}

	public float[] getFloatArray(String key) {
		String[] strings = getAll(key);
		if(strings == null) return null;
		float[] ret = new float[strings.length];
		for(int i=0;i<strings.length;i++) {
			try {
				ret[i] = Float.parseFloat(strings[i]);
			} catch(NumberFormatException e) {
				Logger.error(this, "Cannot parse "+strings[i]+" : "+e,e);
				return null;
			}
		}

		return ret;
	}

    public boolean[] getBooleanArray(String key) {
        String[] strings = getAll(key);
        if(strings == null) return null;
        boolean[] ret = new boolean[strings.length];
        for(int i=0;i<strings.length;i++) {
            try {
                ret[i] = Boolean.parseBoolean(strings[i]);
            } catch(NumberFormatException e) {
                Logger.error(this, "Cannot parse "+strings[i]+" : "+e,e);
                return null;
            }
        }

        return ret;
    }

	public void putOverwrite(String key, String[] strings) {
		putOverwrite(key, unsplit(strings));
	}

    public void putEncoded(String key, String[] strings) {
        String[] copy = Arrays.copyOf(strings, strings.length);
        for(int i=0;i<copy.length;i++) {
            copy[i] = Base64.encodeUTF8(strings[i]);
        }
        putSingle(key, unsplit(copy));
    }

	public String getString(String key) throws FSParseException {
		String s = get(key);
		if(s == null) throw new FSParseException("No such element "+key);
		return s;
	}

	/** Set the headers. This is a list of String's that are written before the name=value pairs.
	 * Usually this is a comment (with each line starting with "#").
	 * @param headers The list of lines to precede the SimpleFieldSet by when we write it.
	 */
	public void setHeader(String... headers) {
		// FIXME LOW should really check that each line doesn't have a "\n" in it
		this.header = headers;
	}

	/** Get the headers. This is a list of String's that are written before the name=value pairs.
     * Usually this is a comment (with each line starting with "#"). */
	public String[] getHeader() {
		return this.header;
	}

	public void put(String key, String[] values) {
	    putSingle(key, unsplit(values));
	}
}
