Flutter AnimatedSize works in one direction only

AnimatedContainer is what you are looking for no need for AnimatedSize.

import 'package:flutter/material.dart';

void main() => runApp(MyApp());

class MyApp extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: 'Flutter Demo',
      theme: ThemeData(
        primarySwatch: Colors.blue,
      ),
      home: MyHomePage(title: 'Flutter Demo Home Page'),
    );
  }
}

class MyHomePage extends StatefulWidget {
  MyHomePage({Key key, this.title}) : super(key: key);
  final String title;

  @override
  _MyHomePageState createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage> with TickerProviderStateMixin {
  double _height = 80.0;
  double _width = 80.0;
  var _color = Colors.blue;
  bool _resized = false;

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      body: Center(
        child: Column(
          mainAxisAlignment: MainAxisAlignment.center,
          children: <Widget>[
            GestureDetector(
              onTap: () {
                setState(() {
                  if (_resized) {
                    _resized = false;
                    _color = Colors.blue;
                    _height = 80.0;
                    _width = 80.0;
                  } else {
                    _resized = true;
                    _color = Colors.blue;
                    _height = 320.0;
                    _width = 320.0;
                  }
                });
              },
              child: AnimatedContainer(
                duration: Duration(seconds: 1),
                width: _width,
                height: _height,
                color: _color,
              ),
            ),
          ],
        ),
      ),
    );
  }
}

Or if you need AnimatedSize you can add a new container with the color as the AnimatedSize parent.

import 'package:flutter/material.dart';

void main() => runApp(MyApp());

class MyApp extends StatelessWidget {
  @override
  Widget build(BuildContext context) {
    return MaterialApp(
      title: 'Flutter Demo',
      theme: ThemeData(
        primarySwatch: Colors.blue,
      ),
      home: MyHomePage(title: 'Flutter Demo Home Page'),
    );
  }
}

class MyHomePage extends StatefulWidget {
  MyHomePage({Key key, this.title}) : super(key: key);
  final String title;

  @override
  _MyHomePageState createState() => _MyHomePageState();
}

class _MyHomePageState extends State<MyHomePage>
    with SingleTickerProviderStateMixin {
  double _height = 320.0;
  double _width = 320.0;
  var _color = Colors.blue;
  bool _resized = false;

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      body: Center(
        child: GestureDetector(
          onTap: () {
            setState(() {
              if (_resized) {
                _resized = false;
                _color = Colors.blue;
                _height = 80.0;
                _width = 80.0;
              } else {
                _resized = true;
                _color = Colors.blue;
                _height = 320.0;
                _width = 320.0;
              }
            });
          },
          child: Container(
            color: _color,
            child: AnimatedSize(
              curve: Curves.easeIn,
              vsync: this,
              duration: Duration(seconds: 1),
              child: Container(
                width: _width,
                height: _height,
              ),
            ),
          ),
        ),
      ),
    );
  }
}

AnimatedContainer isn't really an option when using text, lists or other variably sized widgets because it doesn't allow a width or height of null. Also it won't let you specify an alignment.

Instead I fixed this using a ClipRect with an animation controller. In the animation below you can see the word 'laboris' on the left of the center line in the closing bounce. This shows the text is getting centered during the closing of the container, just keep your eyes on 'laboris'.

I created an AnimatedClipRect widget class that you can easily implement yourself. You can specify alignment, curves, durations and if you want horizontal and/or vertical animation. As it is now it assumes you just want to fully close or open it:

class AnimatedClipRect extends StatefulWidget {
  @override
  _AnimatedClipRectState createState() => _AnimatedClipRectState();

  final Widget child;
  final bool open;
  final bool horizontalAnimation;
  final bool verticalAnimation;
  final Alignment alignment;
  final Duration duration;
  final Duration? reverseDuration;
  final Curve curve;
  final Curve? reverseCurve;

  ///The behavior of the controller when [AccessibilityFeatures.disableAnimations] is true.
  final AnimationBehavior animationBehavior;

  const AnimatedClipRect({
    Key? key,
    required this.child,
    required this.open,
    this.horizontalAnimation = true,
    this.verticalAnimation = true,
    this.alignment = Alignment.center,
    this.duration = const Duration(milliseconds: 500),
    this.reverseDuration,
    this.curve = Curves.linear,
    this.reverseCurve,
    this.animationBehavior = AnimationBehavior.normal,
  }) : super(key: key);
}

class _AnimatedClipRectState extends State<AnimatedClipRect> with TickerProviderStateMixin {
  late AnimationController _animationController;
  late Animation _animation;

  @override
  void dispose() {
     _animationController.dispose();
     super.dispose();
  }

  @override
  void initState() {
    _animationController = AnimationController(
        duration: widget.duration,
        reverseDuration: widget.reverseDuration ?? widget.duration,
        vsync: this,
        value: widget.open ? 1.0 : 0.0,
        animationBehavior: widget.animationBehavior);
    _animation = Tween(begin: 0.0, end: 1.0).animate(CurvedAnimation(
      parent: _animationController,
      curve: widget.curve,
      reverseCurve: widget.reverseCurve ?? widget.curve,
    ));
    super.initState();
  }

  @override
  Widget build(BuildContext context) {
    widget.open ? _animationController.forward() : _animationController.reverse();

    return ClipRect(
      child: AnimatedBuilder(
        animation: _animationController,
        builder: (_, child) {
          return Align(
            alignment: widget.alignment,
            heightFactor: widget.verticalAnimation ? _animation.value : 1.0,
            widthFactor: widget.horizontalAnimation ? _animation.value : 1.0,
            child: child,
          );
        },
        child: widget.child,
      ),
    );
  }
}

Basic usage example, put whatever you want to animate in its child:

// declare bool _open somewhere

Column(
  children: <Widget>[
    AnimatedClipRect(
      open: _open,
      horizontalAnimation: false,
      verticalAnimation: true,
      alignment: Alignment.center,
      duration: const Duration(milliseconds: 1000),
      curve: Curves.bounceOut,
      reverseCurve: Curves.bounceIn,
      child: Container(
        color: Colors.lightGreenAccent,
        padding: const EdgeInsets.all(8),
        child: const Text(
            'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.'),
      ),
    ),
    ElevatedButton(
        child: const Text("open/close"),
        onPressed: () {
          setState(() => _open ^= true);
        }),
  ],
)

As you can see all you have to do is change _open and do setState((){}) to trigger the animation.