Building a library using autotools from cmake

I think that you'd be better off using the ExternalProject feature of cmake. I guess you have your project and have libantrl in a sub directory?

project
      +- libantlr
      +- mysrc
  ---- etc ----

If that's the case, you can do something like this in the top level CMakeLists.txt:

cmake_minimum_required(VERSION 2.8)
project(test)
include(ExternalProject)
ExternalProject_Add(libantlr
    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/libantlr
    CONFIGURE_COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/libantlr/configure --prefix=<INSTALL_DIR>
    BUILD_COMMAND ${MAKE})

The <INSTALL_DIR> is expanded to something like libantlr-prefix, so things are installed in your build tree rather than in /usr/local, which is what autotools would do without a prefix.

I needed to do something similar but found it surprisingly difficult to get a working solution, despite the example provided here with the accepted answer, and code snippets provided in several other blog posts, the CMake support email listserv archives, etc. For the benefit of others who come across this question, here is my solution.

The external project we wanted to use is libmodbus, though I believe my solution is general enough to work with any project configured with the standard autoconf recipe of ./autoconf.sh && configure.sh && make && make install.

We wanted to add libmodbus as a submodule of our git repository. We added to our repository at the path <root>/opt/libmodbus. The CMake code to configure it is located in <root>/cmake/modbus.cmake, which is included from our root CMakeLists.txt using

# libmodbus
include(cmake/modbus.cmake)

The content of cmake/modbus.cmake is:

include(ExternalProject)

set(MODBUS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/opt/libmodbus)
set(MODBUS_BIN ${CMAKE_CURRENT_BINARY_DIR}/libmodbus)
set(MODBUS_STATIC_LIB ${MODBUS_BIN}/lib/libmodbus.a)
set(MODBUS_INCLUDES ${MODBUS_BIN}/include)

file(MAKE_DIRECTORY ${MODBUS_INCLUDES})

ExternalProject_Add(
    libmodbus
    PREFIX ${MODBUS_BIN}
    SOURCE_DIR ${MODBUS_DIR}
    DOWNLOAD_COMMAND cd ${MODBUS_DIR} && git clean -dfX && ${MODBUS_DIR}/autogen.sh
    CONFIGURE_COMMAND ${MODBUS_DIR}/configure --srcdir=${MODBUS_DIR} --prefix=${MODBUS_BIN} --enable-static=yes --disable-shared
    BUILD_COMMAND make
    INSTALL_COMMAND make install
    BUILD_BYPRODUCTS ${MODBUS_STATIC_LIB}
)

add_library(modbus STATIC IMPORTED GLOBAL)

add_dependencies(modbus libmodbus)

set_target_properties(modbus PROPERTIES IMPORTED_LOCATION ${MODBUS_STATIC_LIB})
set_target_properties(modbus PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${MODBUS_INCLUDES})

A component that uses libmodbus can declare its dependency as usual:

    add_executable(hello_modbus main.cpp)
    target_link_libraries(hello_modbus modbus)

A few notes:

1. This abuses the DOWNLOAD_COMMAND to perform the autogen.sh step. The git clean -dfX is probably not necessary (it is a leftover from an earlier version that used the BUILD_IN_SOURCE option. If you really want to download the code instead of using a git submodule, you'll need to modify this line appropriately.
2. We go to the trouble to force a static-only build of the library. Adjust your configure command line if you want shared libraries.
3. The set_target_properties command to set the IMPORTED_LOCATION will fail without the BUILD_BYPRODUCTS ${MODBUS_STATIC_LIB} declaration.
4. Likewise, the set_target_properties command to set the INTERFACE_INCLUDE_DIRECTORIES will fail without the file(MAKE_DIRECTORY ${MODBUS_INCLUDES}).