How to define private constructors in javascript?

One can use a variable (initializing) inside a closure which can throw an error if the constructor was called directly instead of via a class method:

var Score = (function () {
  var initializing = false;

  var Score = function (score, hasPassed) {
      if (!initializing) {
         throw new Error('The constructor is private, please use mkNewScore.');
      }

      initializing = false;
      this.score = score;
      this.hasPassed = hasPassed;
  };

  Score.mkNewScore = function (score) {
      intializing = true;
      return new Score(score, score >= 33);
  };

  return Score;
})();

Is there a solution which will allow me to say x instanceof Score?

Yes. Conceptually, @user2864740 is right, but for instanceof to work we need to expose (return) a function instead of a plain object. If that function has the same .prototype as our internal, private constructor, the instanceof operator does what is expected:

var Score  = (function () {

  // the module API
  function PublicScore() {
    throw new Error('The constructor is private, please use Score.makeNewScore.');
  }

  // The private constructor
  var Score = function (score, hasPassed) {
      this.score = score;
      this.hasPassed = hasPassed;
  };

  // Now use either
  Score.prototype = PublicScore.prototype; // to make .constructor == PublicScore,
  PublicScore.prototype = Score.prototype; // to leak the hidden constructor
  PublicScore.prototype = Score.prototype = {…} // to inherit .constructor == Object, or
  PublicScore.prototype = Score.prototype = {constructor:null,…} // for total confusion :-)

  // The preferred smart constructor
  PublicScore.mkNewScore = function (score) {
      return new Score(score, score >= 33);
  };

  return PublicScore;
}());

> Score.mkNewScore(50) instanceof Score
true
> new Score
Error (…)