Deselect node from JTree when click any place outside the tree

First of all you don't need to subclass the DefaultTreeCellRenderer if you just want to change some colors. You can create a new one, set the colors as you wish and set it to the tree. In the below code sample I've done this in the getDefaultTreeCellRenderer().

Your panel contains two elements the tree and the text area. To achieve what you needed I added a mouse listener and a focus listener to the tree:

• The Mouse Listener - on mouseClicked() is triggered both when you click inside the tree or outside it (but not in the TextArea, for that we have the focus listener). To check whether you clicked in the boundaries of a cell we use the tree.getRowForLocation(e.getX(),e.getY()) and if it returns -1 this means we clicked outside any cell so we can clear the selection
• The Focus Listener - when you lose focus from the JTree and click on the text area this will be triggered and we just clear the selection

We needed both listeners because the first one gets triggered only when you click on the tree and around it but not it the Text area and the second one gets triggered when you focus out of the tree area and focus on the text area.

import javax.swing.*;
import javax.swing.tree.DefaultTreeCellRenderer;
import java.awt.*;
import java.awt.event.FocusEvent;
import java.awt.event.FocusListener;
import java.awt.event.MouseEvent;
import java.awt.event.MouseListener;

public class JTreeSelectDeselect {

    public static void main(String[] args) {
        JFrame frame = new JFrame();
        frame.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);

        JPanel panel = new JPanel(new BorderLayout());
        JTree tree = new JTree();

        tree.setCellRenderer(getDefaultTreeCellRenderer());
        tree.addMouseListener(getMouseListener(tree));
        tree.addFocusListener(getFocusListener(tree));

        panel.add(tree, BorderLayout.LINE_START);
        panel.add(new JScrollPane(new JTextArea(10, 30)));
        frame.add(panel);

        frame.pack();
        frame.setVisible(true);
    }

    private static DefaultTreeCellRenderer getDefaultTreeCellRenderer() {
        DefaultTreeCellRenderer defaultTreeCellRenderer = new DefaultTreeCellRenderer();
        defaultTreeCellRenderer.setBackgroundSelectionColor(new Color(86, 92, 160));
        defaultTreeCellRenderer.setBackgroundNonSelectionColor(new Color(135, 151, 53));
        defaultTreeCellRenderer.setBackground(new Color(225, 225, 221, 255));
        defaultTreeCellRenderer.setForeground(new Color(225, 225, 221, 255));
        return defaultTreeCellRenderer;
    }

    private static FocusListener getFocusListener(final JTree tree) {
        return new FocusListener() {
            @Override
            public void focusGained(FocusEvent e) {

            }

            @Override
            public void focusLost(FocusEvent e) {
                System.out.println("focus lost");
                tree.clearSelection();
            }
        };
    }

    private static MouseListener getMouseListener(final JTree tree) {
        return new MouseListener() {
            @Override
            public void mouseClicked(MouseEvent e) {
                System.out.println("mouse clicked");
                if(tree.getRowForLocation(e.getX(),e.getY()) == -1) {
                    System.out.println("clicked outside a specific cell");
                    tree.clearSelection();
                }
            }

            @Override
            public void mousePressed(MouseEvent e) {

            }

            @Override
            public void mouseReleased(MouseEvent e) {

            }

            @Override
            public void mouseEntered(MouseEvent e) {

            }

            @Override
            public void mouseExited(MouseEvent e) {

            }
        };
    }
}

If you want to deselect all selected nodes in the tree on mouse click somewhere else, you need to get notified when user clicks somewhere else. To get it you can use the global event listener (Toolkit.getDefaultToolkit().addAWTEventListener()).

If you want to display some checkboxes in your tree you need a custom cell renderer, which for a condition returns a checkbox. Also you need a data container to hold whether the node is selected and finally you need a routine to update your data model, when the node is clicked. The last thing is usually provided by cell editors (more about in the article about editors and renderes), but in your case it's easier to do using the global mouse listener, that we've installed before.

Of course my example contains a little bit "Swing magic", so when you don't understand something, feel free to ask me about ;)

import java.awt.AWTEvent;
import java.awt.BorderLayout;
import java.awt.Color;
import java.awt.Component;
import java.awt.Toolkit;
import java.awt.event.AWTEventListener;
import java.awt.event.MouseEvent;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

import javax.swing.JCheckBox;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JScrollPane;
import javax.swing.JTextArea;
import javax.swing.JTree;
import javax.swing.SwingUtilities;
import javax.swing.WindowConstants;
import javax.swing.tree.DefaultTreeCellRenderer;
import javax.swing.tree.DefaultTreeModel;
import javax.swing.tree.TreePath;

/**
 * <code>JTreeDeselected</code>.
 */
public class JTreeSelectDeselect {

    private final JTree tree = new JTree();

    // model to hold nodes that must be presented as check boxes 
    // and whether the check boxes are selected
    private final Map<Object, Boolean> checkMap = new HashMap<>();

    // listener as method reference
    private AWTEventListener awtListener = this::mouseClicked;

    public static void main(String[] args) {
        SwingUtilities.invokeLater(new JTreeSelectDeselect()::start);
    }

    private void start() {
        checkMap.put("football", true);
        checkMap.put("soccer", false);
        checkMap.put("violet", true);
        checkMap.put("yellow", false);
        checkMap.put("pizza", true);
        checkMap.put("ravioli", false);
        JFrame frame = new JFrame();
        frame.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);

        JPanel panel = new JPanel(new BorderLayout());
        // JTree tree = new JTree();
        tree.setCellRenderer(new DeselectTreeCellRenderer(checkMap));
        panel.add(new JScrollPane(tree), BorderLayout.LINE_START);
        panel.add(new JScrollPane(new JTextArea(10, 30)));
        frame.add(panel);
        // register global listener
        Toolkit.getDefaultToolkit().addAWTEventListener(awtListener, AWTEvent.MOUSE_EVENT_MASK);
        frame.pack();
        frame.setLocationRelativeTo(null);
        frame.setVisible(true);
    }

    private void mouseClicked(AWTEvent evt) {
        if (evt instanceof MouseEvent) {
            MouseEvent me = (MouseEvent) evt;
            if (me.getID() == MouseEvent.MOUSE_PRESSED) {
                if (me.getComponent().equals(tree)) {
                    TreePath path = tree.getPathForLocation(me.getX(), me.getY());
                    if (path == null) {
                        tree.clearSelection();
                    } else {
                        // update check box value
                        String pathString = Objects.toString(path.getLastPathComponent(), "");
                        Boolean val = checkMap.get(pathString);
                        if (val != null) {
                            checkMap.put(pathString, !val);
                            ((DefaultTreeModel) tree.getModel()).valueForPathChanged(path, pathString);
                        }
                    }
                } else {
                    tree.clearSelection();
                }
            }
        }
    }
}

class DeselectTreeCellRenderer extends DefaultTreeCellRenderer {

    private final JCheckBox checkBox = new JCheckBox();

    private final Map<Object, Boolean> checkMap;

    public DeselectTreeCellRenderer(Map<Object, Boolean> checkMap) {
        this.checkMap = checkMap;
    }

    @Override
    public Color getBackgroundSelectionColor() {
        return new Color(86, 92, 160);
    }

    @Override
    public Color getBackground() {
        return (null);
    }

    @Override
    public Color getBackgroundNonSelectionColor() {
        return new Color(23, 27, 36);
    }

    @Override
    public Component getTreeCellRendererComponent(JTree tree, Object value, boolean sel, boolean exp, boolean leaf, int row,
            boolean hasFocus) {
        super.getTreeCellRendererComponent(tree, value, sel, exp, leaf, row, hasFocus);

        setForeground(new Color(225, 225, 221, 255));
        setOpaque(false);

        // if our "check model" contains an entry for the node,
        // present the node as check box
        if (checkMap.containsKey(Objects.toString(value))) {
            checkBox.setOpaque(true);
            checkBox.setSelected(Boolean.TRUE == checkMap.get(Objects.toString(value)));
            checkBox.setBackground(sel ? getBackgroundSelectionColor() : getBackgroundNonSelectionColor());
            checkBox.setForeground(getForeground());
            checkBox.setFont(getFont());
            checkBox.setBorder(getBorder());
            checkBox.setText(getText());
            return checkBox;
        }
        return this;
    }
}

The Robot class in the Java AWT package is used to generate native system input events for the purposes of test automation, self-running demos, and other applications where control of the mouse and keyboard is needed. The primary purpose of Robot is to facilitate automated testing of Java platform implementations. In simple terms, the class provides control over the mouse and keyboard devices. In the below snippet "Robot" has been used to capture the event and clear selection from the tree.

import java.awt.AWTException;
import java.awt.Robot;
import java.awt.event.*;
import javax.swing.*;
import javax.swing.tree.TreePath;

public class TreeDeselectionTest {

  JTree createdTreeInstance = new JTree();

  TreePath pathSelectionInstance;

  Robot robotInstance;

  public static void main(String[] args) {
    SwingUtilities.invokeLater(new Runnable() {
      public void run() {
        new TreeDeselectionTest().createTreeAndCaptureEvents();
      }
    });
  }

  public void createTreeAndCaptureEvents() {
    try {
      robotInstance = new Robot();
    } catch (AWTException exceptionInstance) {
      exceptionInstance.printStackTrace();
    }
    createdTreeInstance.setShowsRootHandles(false);
    createdTreeInstance.addMouseListener(new MouseAdapter() {
      @Override
      public void mousePressed(MouseEvent eventMousePressInstance) {
        if (robotInstance != null && pathSelectionInstance != null && eventMousePressInstance.getButton() == MouseEvent.BUTTON1) {
          createdTreeInstance.clearSelection();
          robotInstance.mousePress(InputEvent.BUTTON1_MASK);
          robotInstance.mouseRelease(InputEvent.BUTTON1_MASK);
        }
        pathSelectionInstance = createdTreeInstance.getSelectionPath();
      }
    });
    JFrame frameConetnds = new JFrame();
    frameConetnds.setContentPane(new JScrollPane(createdTreeInstance));
    frameConetnds.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
    frameConetnds.setSize(400, 400);
    frameConetnds.setLocationRelativeTo(null);
    frameConetnds.setVisible(true);
  }
}