why doesn't the frame close when i press the escape key?

The invocation of setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE); does not close the frame, it will define the behaviour when the windows decoration [X] close button is pressed (Which you have disabled for full screen). You could replace this with setVisible(false); or exit your programm.

Use dispose() method.

addKeyListener(new KeyAdapter() {
 public void keyPressed(KeyEvent ke) {  // handler
    if(ke.getKeyCode() == KeyEvent.VK_ESCAPE) {
      System.out.println("escaped ?");
      displayFullScreen.this.dispose();
      } 
     else {
      System.out.println("not escaped");
      }
     } 
});

import java.awt.event.ActionEvent;
import java.awt.event.KeyEvent;
import javax.swing.AbstractAction;
import javax.swing.JComponent;
import javax.swing.JFrame;
import javax.swing.KeyStroke;

public abstract class EscapableFrame extends JFrame
{
    public EscapableFrame()
    {
        // on ESC key close frame
        getRootPane().getInputMap(JComponent.WHEN_IN_FOCUSED_WINDOW).put(
        KeyStroke.getKeyStroke(KeyEvent.VK_ESCAPE, 0), "Cancel"); //$NON-NLS-1$
        getRootPane().getActionMap().put("Cancel", new AbstractAction()
        { 

            public void actionPerformed(ActionEvent e)
            {
                System.exit(0);
                //framename.setVisible(false);
            }
       });

   }

}