Change selection color on view-based NSTableView

Since you're using the view based NSTableView, you can subclass NSTableRowView, feed it to the table delegate method - (NSTableRowView *)tableView:(NSTableView *)tableView rowViewForRow:(NSInteger)row;, then customize your selection in the row view class.

Here's an example:

- (void)drawSelectionInRect:(NSRect)dirtyRect {
    if (self.selectionHighlightStyle != NSTableViewSelectionHighlightStyleNone) {
        NSRect selectionRect = NSInsetRect(self.bounds, 2.5, 2.5);
        [[NSColor colorWithCalibratedWhite:.65 alpha:1.0] setStroke];
        [[NSColor colorWithCalibratedWhite:.82 alpha:1.0] setFill];
        NSBezierPath *selectionPath = [NSBezierPath bezierPathWithRoundedRect:selectionRect xRadius:6 yRadius:6];
        [selectionPath fill];
        [selectionPath stroke];
    }
}

Here is James Chen's solution in Swift 3. I've also added the delegate method.

class MyNSTableRowView: NSTableRowView {

    override func drawSelection(in dirtyRect: NSRect) {
        if self.selectionHighlightStyle != .none {
            let selectionRect = NSInsetRect(self.bounds, 2.5, 2.5)
            NSColor(calibratedWhite: 0.65, alpha: 1).setStroke()
            NSColor(calibratedWhite: 0.82, alpha: 1).setFill()
            let selectionPath = NSBezierPath.init(roundedRect: selectionRect, xRadius: 6, yRadius: 6)
            selectionPath.fill()
            selectionPath.stroke()
        }
    }
}

NSTableViewDelegate:

func tableView(_ tableView: NSTableView, rowViewForRow row: Int) -> NSTableRowView? {
    return MyNSTableRowView()
}