JavaFx TreeView css

how to change font only on selected item in TreeView?

In your css file, just define the font for .tree-cell:selected:

.tree-cell:selected {
    -fx-font-weight: bold ;
}

how to setup background on selected sub-tree?

This one's a little tricky. You want the background of any expanded node, and any node whose parent is not the root, to be different (that's not how you phrased it but I think it is logically equivalent). An expanded node has a CSS pseudoclass already. For the "parent is not the root", you need to define your own pseudoclass:

PseudoClass subElementPseudoClass = PseudoClass.getPseudoClass("sub-tree-item");

Now observe the treeItem property of the cell, and update the pseudoclass state when it changes:

treeView.setCellFactory(tv -> {
    TreeCell<TabMenuElement> cell = new TreeCell<TabMenuElement>() {
        @Override
        public void updateItem(TabMenuElement item, boolean empty) {
            super.updateItem(item, empty);
            setDisclosureNode(null);

            if (empty) {
                setText("");
                setGraphic(null);
            } else {
                setText(item.getName()); // appropriate text for item
                if (item.getIv() != null) {
                    setGraphic(item.getIv());
                }
            }
        }
    };

    cell.treeItemProperty().addListener((obs, oldTreeItem, newTreeItem) -> {
        cell.pseudoClassStateChanged(subElementPseudoClass,
            newTreeItem != null && newTreeItem.getParent() != cell.getTreeView().getRoot());
    }

    return cell ;
});

Now in your CSS file you can do

.tree-cell:expanded, .tree-cell:sub-tree-item {
    -fx-background-color: ... ;
}

how to force that at most one subtree could be expanded

Add the following ChangeListener to each of your TreeItem's expanded properties:

ChangeListener<Boolean> expandedListener = (obs, wasExpanded, isNowExpanded) -> {
    if (isNowExpanded) {
        ReadOnlyProperty<?> expandedProperty = (ReadOnlyProperty<?>) obs ;
        Object itemThatWasJustExpanded = expandedProperty.getBean();
        for (TreeItem<TabMenuElement> item : treeView.getRoot().getChildren()) {
            if (item != itemThatWasJustExpanded) {
                item.setExpanded(false);
            }
        }
    }
};

TreeItem<TabMenuElement> biblioteka = new TreeItem<>(...);
biblioteka.expandedProperty().addListener(expandedListener);
TreeItem<TabMenuElement> analiza = new TreeItem<>(...);
analiza.expandedProperty().addListener(expandedListener);
// etc, for all "top-level" items.

how to set bigger size to top level items?

.tree-cell {
    -fx-padding: 0.75em 0em 0.75em 0em ;
}
.tree-cell:sub-tree-item {
    -fx-padding: 0.25em ;
}

(Or change the font size, or similar.)

Here is a complete example:

import javafx.application.Application;
import javafx.beans.property.ReadOnlyProperty;
import javafx.beans.value.ChangeListener;
import javafx.css.PseudoClass;
import javafx.scene.Scene;
import javafx.scene.control.TreeCell;
import javafx.scene.control.TreeItem;
import javafx.scene.control.TreeView;
import javafx.scene.layout.BorderPane;
import javafx.stage.Stage;

public class StyledUniqueExpandingTree extends Application {

    @Override
    public void start(Stage primaryStage) {
        TreeView<String> tree = new TreeView<>();
        tree.setShowRoot(false);
        TreeItem<String> root = new TreeItem<>("");
        tree.setRoot(root);

        ChangeListener<Boolean> expandedListener = (obs, wasExpanded, isNowExpanded) -> {
            if (isNowExpanded) {
                ReadOnlyProperty<?> expandedProperty = (ReadOnlyProperty<?>) obs ;
                Object itemThatWasJustExpanded = expandedProperty.getBean();
                for (TreeItem<String> item : tree.getRoot().getChildren()) {
                    if (item != itemThatWasJustExpanded) {
                        item.setExpanded(false);
                    }
                }
            }
        };

        for (int i=1; i<=4; i++) {
            TreeItem<String> item = new TreeItem<>("Top level "+i);
            item.expandedProperty().addListener(expandedListener);
            root.getChildren().add(item);
            for (int j=1; j<=4; j++) {
                TreeItem<String> subItem = new TreeItem<>("Sub item "+i+":"+j);
                item.getChildren().add(subItem);
            }
        }

        PseudoClass subElementPseudoClass = PseudoClass.getPseudoClass("sub-tree-item");

        tree.setCellFactory(tv -> {
            TreeCell<String> cell = new TreeCell<String>() {
                @Override
                public void updateItem(String item, boolean empty) {
                    super.updateItem(item, empty);
                    setDisclosureNode(null);

                    if (empty) {
                        setText("");
                        setGraphic(null);
                    } else {
                        setText(item); // appropriate text for item
                    }
                }

            };
            cell.treeItemProperty().addListener((obs, oldTreeItem, newTreeItem) -> {
                cell.pseudoClassStateChanged(subElementPseudoClass,
                        newTreeItem != null && newTreeItem.getParent() != cell.getTreeView().getRoot());
            });
            return cell ;
        });

        BorderPane uiRoot = new BorderPane(tree);
        Scene scene = new Scene(uiRoot, 250, 400);
        scene.getStylesheets().add("styled-unique-expanded-tree.css");
        primaryStage.setScene(scene);
        primaryStage.show();
    }

    public static void main(String[] args) {
        launch(args);
    }
}

with styled-unique-expanded-tree.css:

.tree-view, .tree-cell {
    -fx-background-color: black ;
    -fx-text-fill: white ;
}
.tree-cell:expanded, .tree-cell:sub-tree-item {
    -fx-background-color: #404040 ;
}
.tree-cell:selected {
    -fx-font-weight: bold ;
}
.tree-cell {
    -fx-padding: 0.75em 0em 0.75em 0em ;
}
.tree-cell:sub-tree-item {
    -fx-padding: 0.25em ;
}