How to check, the selected text is bold or not (contenteditable)

The reliable way is to traverse the parent tree checking getComputedStyle.

I'll assume you have the selected element(s) already.

function isBold(_element) {
  var element = _element;
  while (element) {
    var style = getComputedStyle(element).fontWeight;
    if (Number(fontWeight) > 400 || fontWeight === 'bold' || fontWeight === 'bolder') {
      return true;
    }
    element = element.parentNode;
  }
  return false;
}

jQuery(function($) {
    $('.embolden').click(function(){
        if(selectionIsBold()){
          alert('bold');
        }
        else {
          alert('not bold');
        }
    });
});

function selectionIsBold() {
    var isBold = false;
    if (document.queryCommandState) {
        isBold = document.queryCommandState("bold");
    }
    return isBold;
}
.bold {
    font-weight: bold;
}
<div contenteditable="true" class="textEditor">Some <span class="bold">random </span>text.</div>
<a href="#" class="embolden">Is Bold Text</a>
<script
  src="https://code.jquery.com/jquery-3.2.1.min.js"
  integrity="sha256-hwg4gsxgFZhOsEEamdOYGBf13FyQuiTwlAQgxVSNgt4="
  crossorigin="anonymous"></script>

Highlight the text and click on the link.