Sort HTML elements by data-attribute

You could do this with sort method on an array and then append back elements to ul.

let ul = document.querySelector("ul");
let li = ul.querySelectorAll("li");

let sorted = Array.from(li).sort((a, b) => {
  return (b.dataset.categoryGroup == 'jeans') - (a.dataset.categoryGroup == 'jeans')
})

ul.innerHTML = "";
sorted.forEach(e => ul.appendChild(e))
<ul>
  <li class="category-item" data-category-group="jeans">Bottoms</li>
  <li class="category-item" data-category-group="tops">Shirt</li>
  <li class="category-item" data-category-group="jeans">Jeans</li>
  <li class="category-item" data-category-group="jeans">Shorts</li>
  <li class="category-item" data-category-group="tops">Hoodie</li>
  <li class="category-item" data-category-group="accesories">Sunglasses</li>
</ul>

You should use Array#sort, just pass it a function that checks the data-category-group property (using String#localeCompare()):

var categoryItems = document.querySelectorAll("[data-category-group]");
var categoryItemsArray = Array.from(categoryItems);

let sorted = categoryItemsArray.sort(sorter);

function sorter(a,b) {
    return a.dataset.categoryGroup.localeCompare(b.dataset.categoryGroup); // sorts based on alphabetical order
}

document.querySelector("button").onclick = () => sorted.forEach(e => document.querySelector("#demo").appendChild(e));
<ul id="demo">
  <li class="category-item" data-category-group="jeans">Bottoms</li>
  <li class="category-item" data-category-group="tops">Shirt</li>
  <li class="category-item" data-category-group="jeans">Jeans</li>
  <li class="category-item" data-category-group="jeans">Shorts</li>
  <li class="category-item" data-category-group="tops">Hoodie</li>
  <li class="category-item" data-category-group="accesories">Sunglasses</li>
</ul>


<button>Sort Elements</button>