How to calculate and export tasseled caps from MODIS collection in GEE?

I think you were close to a solution. You defined the coefficients as a 7-band image, you just needed to multiply the reflectance data by the coefficients and sum the weighted bands. Note if you multiply a n-band image by a n-band image then the arithmetic will be applied per band which is exactly what we would want with the tasseled cap transform.

Here is the code to apply a Tasseled Cap transform using images instead of arrays:

// get the BRDF corrected dataset
var imageCollection = ee.ImageCollection("MODIS/006/MCD43A4")
  .filterDate('2005-01-01','2006-01-01') // filter to one year to prevent memory errors
  .filter(ee.Filter.calendarRange(5,9,'month'));

// use regex to select the bands (much cleaner)
var modbands = imageCollection.select("^(Nadir|BRDF).*");

// function to mask out of range values from dataset
var qas = function(image){ 
  var mask1 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band1").eq(0);
  var mask2 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band2").eq(0);
  var mask3 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band3").eq(0);
  var mask4 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band4").eq(0);
  var mask5 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band5").eq(0);
  var mask6 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band6").eq(0);
  var mask7 = image.select("BRDF_Albedo_Band_Mandatory_Quality_Band7").eq(0);

  return image.updateMask(mask1).updateMask(mask2).updateMask(mask3).updateMask(mask4).updateMask(mask5).updateMask(mask6).updateMask(mask7);
};

// mask all images in the collection based on quality bands
var masked= modbands.map(qas);

// tasseled cap function
function tasseledcap(img){
  // define the TC coefficients as 7-band images
  var brightCoefs = ee.Image([0.4395,0.5945,0.2460,0.3918,0.3506,0.2136,0.2678]);
  var greenCoefs = ee.Image([-0.4064,0.5129,-0.2744,-0.2893,0.4882,-0.0036,-0.4169]);
  var wetCoefs = ee.Image([0.1147,0.2489,0.2408,0.3132,-0.3122,-0.6416,-0.5087]);

  // multiply the Nadir reflectance values by the coefficients and sum across the weigted values
  // note that the multiplication is applied per band, band1 * band1 band2 * band2 ... bandn * bandn, band1 * band1 band2 * band2 ... bandn * bandn
  var bright = img.multiply(brightCoefs).reduce(ee.Reducer.sum()).rename('brightness');
  var green = img.multiply(greenCoefs).reduce(ee.Reducer.sum()).rename('greenness');
  var wet = img.multiply(wetCoefs).reduce(ee.Reducer.sum()).rename('wetness');

  // concat the transformed images into one image
  var tcImg = ee.Image.cat([bright,green,wet]).copyProperties(img,['system:time_start']);
  return tcImg;
}

// select the Nadir reflectance bands and apply the tasseled cap transform
var tcColl = masked.select('Nadir.*').map(tasseledcap);

Map.addLayer(masked.median(),{min:50,max:5500,gamma:1.5,bands:'Nadir_Reflectance_Band7,Nadir_Reflectance_Band2,Nadir_Reflectance_Band3'},'BRDF Reflectance');
Map.addLayer(tcColl.median(),{min:-4000,max:8000},'Tasseled Cap Transform');

I did not check to make sure the coefficients were right so please be careful of that...here is a link to the code editor: https://code.earthengine.google.com/fcfe784a7da2ea5235f2c4cab21e7cb2