MTOM Encoding and Custom binding

You have to add message encoding element:

  <customBinding> 
    <binding name="stsBinding"> 
      <security authenticationMode="UserNameOverTransport"  
                requireDerivedKeys="false" 
                keyEntropyMode="ServerEntropy"  
                requireSecurityContextCancellation="false" 
                requireSignatureConfirmation="false"> 
      </security> 
      <mtomMessageEncoding />
      <httpsTransport /> 
    </binding> 
  </customBinding>