javascript atob returning 'String contains an invalid character'

atob breaks due to the newlines in the response. Remove them to get your code to work:

function addSVGToPage(SVGToAdd) {
    var entry, decodedEntry;         // <-- What is this doing here? It's unused.
    makeAJAXCall(SVGToAdd, function (returnedJSON) {
        console.info(window.atob(returnedJSON.data.content.replace(/\s/g, '')));
        //                                                ^^^^^^^^^^^^^^^^^^^
    });
}

According to MDN docs, you might need to escape and then decodeURIComponent to handle unicode:

function utf8_to_b64( str ) {
    return window.btoa(unescape(encodeURIComponent( str )));
}

function b64_to_utf8( str ) {
    return decodeURIComponent(escape(window.atob( str )));
}

// Usage:
utf8_to_b64('✓ à la mode'); // "4pyTIMOgIGxhIG1vZGU="
b64_to_utf8('4pyTIMOgIGxhIG1vZGU='); // "✓ à la mode"