Transitioning between view controller, OS X

I think the simplest way is that swapping contentViewController of NSWindow.

// in NSViewController's subclass
@IBAction func someAction(sender: AnyObject) {
    let nextViewController = ... // instantiate from storyboard or elsewhere

    if let window = view.window where window.styleMask & NSFullScreenWindowMask > 0 {
        // adjust view size to current window
        nextViewController.view.frame = CGRectMake(0, 0, window.frame.width, window.frame.height)
    }

    view.window?.contentViewController = nextViewController
}

This is option #1.

If you want to use segue, create custom one and set it to segue class with identifier in IB.

class ReplaceSegue: NSStoryboardSegue {
    override func perform() {
        if let fromViewController = sourceController as? NSViewController {
            if let toViewController = destinationController as? NSViewController {
                // no animation.
                fromViewController.view.window?.contentViewController = toViewController
            }
        }
    }
}

This is option #2.

Last option is using presentViewController:animator: of NSViewController. The code below is custom NSViewControllerPresentationAnimator for dissolve animation.

class ReplacePresentationAnimator: NSObject, NSViewControllerPresentationAnimator {
    func animatePresentationOfViewController(viewController: NSViewController, fromViewController: NSViewController) {
        if let window = fromViewController.view.window {
            NSAnimationContext.runAnimationGroup({ (context) -> Void in
                fromViewController.view.animator().alphaValue = 0
            }, completionHandler: { () -> Void in
                viewController.view.alphaValue = 0
                window.contentViewController = viewController
                viewController.view.animator().alphaValue = 1.0
            })
        }
    }

    func animateDismissalOfViewController(viewController: NSViewController, fromViewController: NSViewController) {
        if let window = viewController.view.window {
            NSAnimationContext.runAnimationGroup({ (context) -> Void in
                viewController.view.animator().alphaValue = 0
                }, completionHandler: { () -> Void in
                    fromViewController.view.alphaValue = 0
                    window.contentViewController = fromViewController
                    fromViewController.view.animator().alphaValue = 1.0
            })
        }        
    }
}

Then present VC like this.

@IBAction func replaceAction(sender: AnyObject) {
    let nextViewController = ... // instantiate from storyboard or elsewhere
    presentViewController(nextViewController, animator: ReplacePresentationAnimator())
}

For dismissal, call presentingViewController's dismissViewController: in the presented VC.

@IBAction func dismissAction(sender: AnyObject) {
    presentingViewController?.dismissViewController(self)    
}

Swift4 Version

class ReplacePresentationAnimator: NSObject, NSViewControllerPresentationAnimator {
func animatePresentation(of viewController: NSViewController, from fromViewController: NSViewController) {
    if let window = fromViewController.view.window {
        NSAnimationContext.runAnimationGroup({ (context) -> Void in
            fromViewController.view.animator().alphaValue = 0
        }, completionHandler: { () -> Void in
            viewController.view.alphaValue = 0
            window.contentViewController = viewController
            viewController.view.animator().alphaValue = 1.0
        })
    }
}

func animateDismissal(of viewController: NSViewController, from fromViewController: NSViewController) {
    if let window = viewController.view.window {
        NSAnimationContext.runAnimationGroup({ (context) -> Void in
            viewController.view.animator().alphaValue = 0
        }, completionHandler: { () -> Void in
            fromViewController.view.alphaValue = 0
            window.contentViewController = fromViewController
            fromViewController.view.animator().alphaValue = 1.0
        })
    }
}

}

Hope this help.

If you have one parent view controller, you can assign child view controllers to it, and use the transition method. Example code, to be placed in viewDidLoad of the parent view controller:

if let firstController = self.storyboard?.instantiateController(withIdentifier: "firstController") as? NSViewController {
    firstController.view.autoresizingMask = [.width, .height]
    firstController.view.frame = self.view.bounds
    self.addChild(firstController)
    self.view.addSubview(firstController.view)
}

if let secondController = self.storyboard?.instantiateController(withIdentifier: "secondController") as? NSViewController {
    self.addChild(secondController)
}

DispatchQueue.main.asyncAfter(deadline: .now() + 3) {
    if let firstController = self.children.first, let secondController = self.children.last {
        self.transition(from: firstController, to: secondController, options: .crossfade, completionHandler: nil)
    }
}

It's essential that the view of the first child controller is being added as sub view to the view of the parent controller, otherwise the transition method doesn't work.

In the example above, a storyboard is used with one main view controller (= self), one child view controller with storyboard ID "firstController', and another child view controller with storyboard ID "secondController'