How to append to AES encrypted file
If you're using AES in CBC mode, you can use the second to last block as the IV to decrypt the last block, which may be only partially full, then again to encrypt the plaintext of the last block followed by the new plaintext.
Here's a proof of concept:
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
public class AppendAES {
public static void appendAES(File file, byte[] data, byte[] key) throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException {
RandomAccessFile rfile = new RandomAccessFile(file,"rw");
byte[] iv = new byte[16];
byte[] lastBlock = null;
if (rfile.length() % 16L != 0L) {
throw new IllegalArgumentException("Invalid file length (not a multiple of block size)");
} else if (rfile.length() == 16) {
throw new IllegalArgumentException("Invalid file length (need 2 blocks for iv and data)");
} else if (rfile.length() == 0L) {
// new file: start by appending an IV
new SecureRandom().nextBytes(iv);
rfile.write(iv);
// we have our iv, and there's no prior data to reencrypt
} else {
// file length is at least 2 blocks
rfile.seek(rfile.length()-32); // second to last block
rfile.read(iv); // get iv
byte[] lastBlockEnc = new byte[16];
// last block
// it's padded, so we'll decrypt it and
// save it for the beginning of our data
rfile.read(lastBlockEnc);
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key,"AES"), new IvParameterSpec(iv));
lastBlock = cipher.doFinal(lastBlockEnc);
rfile.seek(rfile.length()-16);
// position ourselves to overwrite the last block
}
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, new SecretKeySpec(key,"AES"), new IvParameterSpec(iv));
byte[] out;
if (lastBlock != null) { // lastBlock is null if we're starting a new file
out = cipher.update(lastBlock);
if (out != null) rfile.write(out);
}
out = cipher.doFinal(data);
rfile.write(out);
rfile.close();
}
public static void decryptAES(File file, OutputStream out, byte[] key) throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException {
// nothing special here, decrypt as usual
FileInputStream fin = new FileInputStream(file);
byte[] iv = new byte[16];
if (fin.read(iv) < 16) {
throw new IllegalArgumentException("Invalid file length (needs a full block for iv)");
};
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, new SecretKeySpec(key,"AES"), new IvParameterSpec(iv));
byte[] buff = new byte[1<<13]; //8kiB
while (true) {
int count = fin.read(buff);
if (count == buff.length) {
out.write(cipher.update(buff));
} else {
out.write(cipher.doFinal(buff,0,count));
break;
}
}
fin.close();
}
public static void main(String[] args) throws Exception {
byte[] key = new byte[]{0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
for (int i = 0; i<1000; i++) {
appendAES(new File("log.aes"),"All work and no play makes Jack a dull boy. ".getBytes("UTF-8"),key);
}
decryptAES(new File("log.aes"), new FileOutputStream("plain.txt"), key);
}
}
I'd like to point out that the output is no different than what would be produced by encrypting all in one run. This is not a custom form of encryption --- it's standard AES/CBC/PKCS5Padding. The only implementation-specific detail is that, in the case of a blank file, I've written the iv before beginning the data.
EDIT: Improved (for my taste) solution using CipherOutputStream:
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.RandomAccessFile;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.CipherOutputStream;
import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
public class AppendAES {
public static CipherOutputStream appendAES(File file, SecretKeySpec key) throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException {
return appendAES(file, key, null);
}
public static CipherOutputStream appendAES(File file, SecretKeySpec key, SecureRandom sr) throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException {
RandomAccessFile rfile = new RandomAccessFile(file,"rw");
byte[] iv = new byte[16];
byte[] lastBlock = null;
if (rfile.length() % 16L != 0L) {
throw new IllegalArgumentException("Invalid file length (not a multiple of block size)");
} else if (rfile.length() == 16) {
throw new IllegalArgumentException("Invalid file length (need 2 blocks for iv and data)");
} else if (rfile.length() == 0L) {
// new file: start by appending an IV
if (sr == null) sr = new SecureRandom();
sr.nextBytes(iv);
rfile.write(iv);
} else {
// file length is at least 2 blocks
rfile.seek(rfile.length()-32);
rfile.read(iv);
byte[] lastBlockEnc = new byte[16];
rfile.read(lastBlockEnc);
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
lastBlock = cipher.doFinal(lastBlockEnc);
rfile.seek(rfile.length()-16);
}
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.ENCRYPT_MODE, key, new IvParameterSpec(iv));
byte[] out;
if (lastBlock != null) {
out = cipher.update(lastBlock);
if (out != null) rfile.write(out);
}
CipherOutputStream cos = new CipherOutputStream(new FileOutputStream(rfile.getFD()),cipher);
return cos;
}
public static CipherInputStream decryptAES(File file, SecretKeySpec key) throws IOException, NoSuchAlgorithmException, NoSuchPaddingException, InvalidKeyException, InvalidAlgorithmParameterException, IllegalBlockSizeException, BadPaddingException {
FileInputStream fin = new FileInputStream(file);
byte[] iv = new byte[16];
if (fin.read(iv) < 16) {
throw new IllegalArgumentException("Invalid file length (needs a full block for iv)");
};
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
CipherInputStream cis = new CipherInputStream(fin,cipher);
return cis;
}
public static void main(String[] args) throws Exception {
byte[] keyBytes = new byte[]{
0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
};
SecretKeySpec key = new SecretKeySpec(keyBytes,"AES");
for (int i = 0; i<100; i++) {
CipherOutputStream cos = appendAES(new File("log.aes"),key);
cos.write("All work and no play ".getBytes("UTF-8"));
cos.write("makes Jack a dull boy. \n".getBytes("UTF-8"));
cos.close();
}
CipherInputStream cis = decryptAES(new File("log.aes"), key);
BufferedReader bread = new BufferedReader(new InputStreamReader(cis,"UTF-8"));
System.out.println(bread.readLine());
cis.close();
}
}
I like the solution provided by maybeWeCouldStealAVan. But this did not correctly implement 'flush()', and I found it was necessary to close and reopen the file each time you append a message, to be sure you don't lose anything. So I rewrote it. My solution will write out the last block each time you flush, but then rewrite this block when the next message is added. With this 2-steps-forward, 1-step-back method, it's not possible to use OutputStream's, instead I've implemented it directly on top of RandomAccessFile.
import javax.crypto.*;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.security.*;
public class FlushableCipherOutputStream extends OutputStream
{
private static int HEADER_LENGTH = 16;
private SecretKeySpec key;
private RandomAccessFile seekableFile;
private boolean flushGoesStraightToDisk;
private Cipher cipher;
private boolean needToRestoreCipherState;
/** the buffer holding one byte of incoming data */
private byte[] ibuffer = new byte[1];
/** the buffer holding data ready to be written out */
private byte[] obuffer;
/** Each time you call 'flush()', the data will be written to the operating system level, immediately available
* for other processes to read. However this is not the same as writing to disk, which might save you some
* data if there's a sudden loss of power to the computer. To protect against that, set 'flushGoesStraightToDisk=true'.
* Most people set that to 'false'. */
public FlushableCipherOutputStream(String fnm, SecretKeySpec _key, boolean append, boolean _flushGoesStraightToDisk)
throws IOException
{
this(new File(fnm), _key, append,_flushGoesStraightToDisk);
}
public FlushableCipherOutputStream(File file, SecretKeySpec _key, boolean append, boolean _flushGoesStraightToDisk)
throws IOException
{
super();
if (! append)
file.delete();
seekableFile = new RandomAccessFile(file,"rw");
flushGoesStraightToDisk = _flushGoesStraightToDisk;
key = _key;
try {
cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
byte[] iv = new byte[16];
byte[] headerBytes = new byte[HEADER_LENGTH];
long fileLen = seekableFile.length();
if (fileLen % 16L != 0L) {
throw new IllegalArgumentException("Invalid file length (not a multiple of block size)");
} else if (fileLen == 0L) {
// new file
// You can write a 16 byte file header here, including some file format number to represent the
// encryption format, in case you need to change the key or algorithm. E.g. "100" = v1.0.0
headerBytes[0] = 100;
seekableFile.write(headerBytes);
// Now appending the first IV
SecureRandom sr = new SecureRandom();
sr.nextBytes(iv);
seekableFile.write(iv);
cipher.init(Cipher.ENCRYPT_MODE, key, new IvParameterSpec(iv));
} else if (fileLen <= 16 + HEADER_LENGTH) {
throw new IllegalArgumentException("Invalid file length (need 2 blocks for iv and data)");
} else {
// file length is at least 2 blocks
needToRestoreCipherState = true;
}
} catch (InvalidKeyException e) {
throw new IOException(e.getMessage());
} catch (NoSuchAlgorithmException e) {
throw new IOException(e.getMessage());
} catch (NoSuchPaddingException e) {
throw new IOException(e.getMessage());
} catch (InvalidAlgorithmParameterException e) {
throw new IOException(e.getMessage());
}
}
/**
* Writes one _byte_ to this output stream.
*/
public void write(int b) throws IOException {
if (needToRestoreCipherState)
restoreStateOfCipher();
ibuffer[0] = (byte) b;
obuffer = cipher.update(ibuffer, 0, 1);
if (obuffer != null) {
seekableFile.write(obuffer);
obuffer = null;
}
}
/** Writes a byte array to this output stream. */
public void write(byte data[]) throws IOException {
write(data, 0, data.length);
}
/**
* Writes <code>len</code> bytes from the specified byte array
* starting at offset <code>off</code> to this output stream.
*
* @param data the data.
* @param off the start offset in the data.
* @param len the number of bytes to write.
*/
public void write(byte data[], int off, int len) throws IOException
{
if (needToRestoreCipherState)
restoreStateOfCipher();
obuffer = cipher.update(data, off, len);
if (obuffer != null) {
seekableFile.write(obuffer);
obuffer = null;
}
}
/** The tricky stuff happens here. We finalise the cipher, write it out, but then rewind the
* stream so that we can add more bytes without padding. */
public void flush() throws IOException
{
try {
if (needToRestoreCipherState)
return; // It must have already been flushed.
byte[] obuffer = cipher.doFinal();
if (obuffer != null) {
seekableFile.write(obuffer);
if (flushGoesStraightToDisk)
seekableFile.getFD().sync();
needToRestoreCipherState = true;
}
} catch (IllegalBlockSizeException e) {
throw new IOException("Illegal block");
} catch (BadPaddingException e) {
throw new IOException("Bad padding");
}
}
private void restoreStateOfCipher() throws IOException
{
try {
// I wish there was a more direct way to snapshot a Cipher object, but it seems there's not.
needToRestoreCipherState = false;
byte[] iv = cipher.getIV(); // To help avoid garbage, re-use the old one if present.
if (iv == null)
iv = new byte[16];
seekableFile.seek(seekableFile.length() - 32);
seekableFile.read(iv);
byte[] lastBlockEnc = new byte[16];
seekableFile.read(lastBlockEnc);
cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
byte[] lastBlock = cipher.doFinal(lastBlockEnc);
seekableFile.seek(seekableFile.length() - 16);
cipher.init(Cipher.ENCRYPT_MODE, key, new IvParameterSpec(iv));
byte[] out = cipher.update(lastBlock);
assert out == null || out.length == 0;
} catch (Exception e) {
throw new IOException("Unable to restore cipher state");
}
}
public void close() throws IOException
{
flush();
seekableFile.close();
}
}
You can see how to use it and test it using:
import org.junit.Test;
import javax.crypto.Cipher;
import javax.crypto.CipherInputStream;
import javax.crypto.spec.IvParameterSpec;
import javax.crypto.spec.SecretKeySpec;
import java.io.*;
import java.io.BufferedWriter;
public class TestFlushableCipher {
private static byte[] keyBytes = new byte[]{
// Change these numbers lest other StackOverflow readers can read your log files
-53, 93, 59, 108, -34, 17, -72, -33, 126, 93, -62, -50, 106, -44, 17, 55
};
private static SecretKeySpec key = new SecretKeySpec(keyBytes,"AES");
private static int HEADER_LENGTH = 16;
private static BufferedWriter flushableEncryptedBufferedWriter(File file, boolean append) throws Exception
{
FlushableCipherOutputStream fcos = new FlushableCipherOutputStream(file, key, append, false);
return new BufferedWriter(new OutputStreamWriter(fcos, "UTF-8"));
}
private static InputStream readerEncryptedByteStream(File file) throws Exception
{
FileInputStream fin = new FileInputStream(file);
byte[] iv = new byte[16];
byte[] headerBytes = new byte[HEADER_LENGTH];
if (fin.read(headerBytes) < HEADER_LENGTH)
throw new IllegalArgumentException("Invalid file length (failed to read file header)");
if (headerBytes[0] != 100)
throw new IllegalArgumentException("The file header does not conform to our encrypted format.");
if (fin.read(iv) < 16) {
throw new IllegalArgumentException("Invalid file length (needs a full block for iv)");
}
Cipher cipher = Cipher.getInstance("AES/CBC/PKCS5Padding");
cipher.init(Cipher.DECRYPT_MODE, key, new IvParameterSpec(iv));
return new CipherInputStream(fin,cipher);
}
private static BufferedReader readerEncrypted(File file) throws Exception
{
InputStream cis = readerEncryptedByteStream(file);
return new BufferedReader(new InputStreamReader(cis));
}
@Test
public void test() throws Exception {
File zfilename = new File("c:\\WebEdvalData\\log.x");
BufferedWriter cos = flushableEncryptedBufferedWriter(zfilename, false);
cos.append("Sunny ");
cos.append("and green. \n");
cos.close();
int spaces=0;
for (int i = 0; i<10; i++) {
cos = flushableEncryptedBufferedWriter(zfilename, true);
for (int j=0; j < 2; j++) {
cos.append("Karelia and Tapiola" + i);
for (int k=0; k < spaces; k++)
cos.append(" ");
spaces++;
cos.append("and other nice things. \n");
cos.flush();
tail(zfilename);
}
cos.close();
}
BufferedReader cis = readerEncrypted(zfilename);
String msg;
while ((msg=cis.readLine()) != null) {
System.out.println(msg);
}
cis.close();
}
private void tail(File filename) throws Exception
{
BufferedReader infile = readerEncrypted(filename);
String last = null, secondLast = null;
do {
String msg = infile.readLine();
if (msg == null)
break;
if (! msg.startsWith("}")) {
secondLast = last;
last = msg;
}
} while (true);
if (secondLast != null)
System.out.println(secondLast);
System.out.println(last);
System.out.println();
}
}
AES is a block cipher. that means that it doesn't encrypt a message character by character, but saves data until it has a chunk of a certain size, then writes it. so that in itself is going to cause problems for you, because your log messages are not likely to match the block size. that's the first problem.
second problem is that "AES" by itself is not a full description of what you are doing. a block cipher can be used in different "modes" (see this good description at wikipedia). many of those modes mix information from earlier in the stream with data that comes later. this makes the encryption more secure, but again is going to cause problems (because you need to store the information that will be mixed between closing and opening the file).
to solve the first problem you want a stream cipher. like you'd expect from the name, this works on a stream of data. now it turns out that some of the cipher modes described above can make a block cipher work like a stream one.
but a stream cipher probably won't help solve the second problem - for that you need to store, somewhere, the data that needs to be carried across between uses so that you can initialise the appended stream correctly.
really, if you're asking all this, how sure are you going to be that the final result is secure? there are lots of mistakes that you might make, even with the above as guidance. i would suggest either finding an existing library that does this, or reducing your requirements so that you are solving a simpler problem (do you really need to append - could you not start a new file in that case? or, like suggested above, add some kind of marker to the file so you can find the different sections?)