How to pass prototype function?

The problem is that you're passing a function a reference to another function, and the passed function is therefore losing scope! Here's the offending line:

Circle.prototype.increaseRadiusBy = function(number) {
    this.r = sumWithFunction(this.getRadius, number);
}

JavaScript objects are in some ways simpler than they appear. When you added the getRadius method to the Circle prototype, you were not defining a class method like you would in classical OO. You were simply defining a named property of the prototype, and assigning a function to the value of that property. When you pass this.getRadius as an argument to a static function, like sumWithFunction, the context of this is lost. It executes with the this keyword bound to window, and since window has no r property, the browser throws an undefined error.

Put another way, the statement this.getRadius() is actually saying "execute the function assigned to the getRadius property of this, and execute it in the context of this. Without calling the function explicitly through that statement, the context is not assigned.

A common solution to this is to add an expected argument to any function which receives another function, for context.

function sumWithFunction(func, context, number) {
    return func.apply(context) + number;
}

function Circle(X, Y, R) {
    this.x = X;
    this.y = Y;
    this.r = R;
}
Circle.prototype.getRadius = function () {
    return this.r;
}
Circle.prototype.increaseRadiusBy = function(number) {
    this.r = sumWithFunction(this.getRadius, this, number);
}

function addFivetoIt(func, context) {
    func.apply(context,[5]);
}

var MyCircle = new Circle(0, 0, 10);
addFivetoIt(MyCircle.increaseRadiusBy, myCircle);

A simpler, but less robust solution would be to declare a function inline that can access a context reference in the local closure.

function sumWithFunction(func, number) {
    return func() + number;
}

function Circle(X, Y, R) {
    this.x = X;
    this.y = Y;
    this.r = R;
}
Circle.prototype.getRadius = function () {
    return this.r;
}
Circle.prototype.increaseRadiusBy = function(number) {
    var me = this;
    this.r = sumWithFunction(function() {
        return me.getRadius()
    }, number);
}

function addFivetoIt(func) {
    func(5);
}

var MyCircle = new Circle(0, 0, 10);
addFivetoIt(function(number) {
    return MyCircle.increaseRadiusBy(number);
});

But by far the simplest solution is to use a newer feature of ECMAScript, a function method called bind. It is explained well here, including the fact that it is not supported by all browsers. That's why a lot of libraries, like jQuery, Prototype, etc., have cross-browser function-binding utility methods like $.proxy.

function sumWithFunction(func, number) {
    return func() + number;
}

function Circle(X, Y, R) {
    this.x = X;
    this.y = Y;
    this.r = R;
}
Circle.prototype.getRadius = function () {
    return this.r;
}
Circle.prototype.increaseRadiusBy = function(number) {
    this.r = sumWithFunction(this.getRadius.bind(this), number); // or $.proxy(this.getRadius,this)
}

function addFivetoIt(func) {
    func(5);
}

var MyCircle = new Circle(0, 0, 10);
addFivetoIt(MyCircle.increaseRadiusBy.bind(MyCircle)); // or $.proxy(MyCircle.increaseRadiusBy,MyCircle)