TableViewCell animation in swift

What you need is an ease out back animation. For more information check out http://easings.net

You can make parametric animations using this library https://github.com/jjackson26/JMJParametricAnimation/tree/master/JMJParametricAnimationDemo

For now, the effect you are trying to do can be roughly done using the code below. You have to do the scaling animation one after another. The way you have done makes all animations start together.
Adding the next animation code in the completion block starts it after the animation. You can further tweak the timings to get the rough effect you need.

    cell.layer.transform = CATransform3DMakeScale(0.1,0.1,1)
    UIView.animateWithDuration(0.3, animations: {
        cell.layer.transform = CATransform3DMakeScale(1.05,1.05,1)
        },completion: { finished in
            UIView.animateWithDuration(0.1, animations: {
                cell.layer.transform = CATransform3DMakeScale(1,1,1)
            })
    })

Swift 4

func tableView(_ tableView: UITableView, willDisplay cell: UITableViewCell, forRowAt indexPath: IndexPath) {
    cell.transform = CGAffineTransform(scaleX: 0.8, y: 0.8)
    UIView.animate(withDuration: 0.4) {
        cell.transform = CGAffineTransform.identity
    }
}

Swift 3 version works like charm!

cell.layer.transform = CATransform3DMakeScale(0.1, 0.1, 1)
    UIView.animate(withDuration: 0.3, animations: {
        cell.layer.transform = CATransform3DMakeScale(1.05, 1.05, 1)
    },completion: { finished in
        UIView.animate(withDuration: 0.1, animations: {
            cell.layer.transform = CATransform3DMakeScale(1, 1, 1)
        })
    })