How can I stop other attributes changing my swatch image?

Facing with the same problem, today I wrote the following fix, it works both with attribute selectors and configurable swatches.

$j(document).ready(function() {
    if (typeof ConfigurableMediaImages === 'undefined' || typeof optionsPrice === 'undefined' || typeof Product === 'undefined') return;

    /**
     * Returns true, if there is a label-matching image on config product for the selected option's label
     * @param el
     * @returns {boolean}
     */
    var matchingImageExists = function(el) {
        // copy from product-media.js:150-152
        var select = $j(el);
        var label = select.find('option:selected').attr('data-label');
        var productId = optionsPrice.productId; //get product ID from options price object

        // the inspection, based on product-media.js:94
        return ConfigurableMediaImages.productImages[productId]['option_labels'][label]['configurable_product'][ConfigurableMediaImages.imageType];
    };

    // activation for attribute selectors (selectors as used in product-media.js:175)
    $j('.product-options .super-attribute-select')
        .off('change', ConfigurableMediaImages.updateImage(this))
        .on('change', function() { if (matchingImageExists(this)) { ConfigurableMediaImages.updateImage(this); } });

    // activation for swatches (overriding Product.ConfigurableSwatches.prototype.updateSelect method in swatches-product.js:722)
    Product.ConfigurableSwatches.prototype.updateSelect = function(attr) {
        // fire select change event
        // this will trigger the validation of the select
        // only fire if this attribute has had a selected option at one time
        if (attr._e.selectedOption !== false && attr._e.optionSelect) {
            this._F.nativeSelectChange = false;
            if (matchingImageExists(attr._e.optionSelect)) ConfigurableMediaImages.updateImage(attr._e.optionSelect);
            this.productConfig.handleSelectChange(attr._e.optionSelect);
            this._F.nativeSelectChange = true;
        };
    }
});

You can include it in a separate JS (update: for this, check dawhoo's answer below), so don't need to modify any of the core files.

matchingImageExists() checks, whether there's an image set in the configurable product with the selected option.

In the last two blocks we are overriding the core attribute selector event handler and the Product.ConfigurableSwatches.prototype.updateSelect method for placing the matchingImageExists() control, so we let Magento change the product images only in that case if a label-matching image exists on config product for the selected option's label.

Hope it'll help you. Please notify in case of bugs as I'll use this fix in my shop as well. :-)

Quick and easy way.

Use the code from @bencergazda and create a file called something like "fixswap.js"

Upload fixswap.js to your /skin/frontend/[THEME]/default/js

Now, go to /app/design/frontend/[THEME]/default/layout and add to your local.xml file

<catalog_product_view>
    <reference name="head">
        <action method="addJs"><script>fixswap.js</script></action>
    </reference>
</catalog_product_view>

If you don't already have a local.xml, create one by using the code below.

<?xml version="1.0"?>
<layout>

    <catalog_product_view>
        <reference name="head">
            <action method="addJs"><script>fixswap.js</script></action>
        </reference>
    </catalog_product_view>

</layout>