How to use Dart analyzer to generate an AST from source and work with the AST?

Wiki

Note: Code samples posted here are not great in terms of quality.
There should be more succinct and efficient way, though so far I've failed to find them.

For further details, search dart-sdk for some function/class names on github.
These were helpful:
https://github.com/dart-lang/sdk/blob/dd0c42cb72a4c218e4b04890f0ef666c6f6c0eb6/pkg/analyzer/test/dart/ast/visitor_test.dart
https://github.com/dart-lang/sdk/blob/71e7ed86c060ff2d695777d2c68c20e25cfb8ef9/pkg/analysis_server/lib/src/services/correction/util.dart
https://github.com/dart-lang/sdk/blob/6715573c6eb5e5a3c2b8257fef02e88a7d707d9f/pkg/analyzer/lib/src/generated/incremental_resolver.dart

For AstVisitor(There are several):https://www.dartdocs.org/documentation/analyzer_experimental/0.8.0/analyzer/RecursiveASTVisitor-class.html

As Günter Zöchbauer mentioned, it's work in progress and poorly documented while it's must-know if you are writing a transformer for compile time code modification.

#Update for new version

import 'package:analyzer/dart/analysis/utilities.dart';
import 'package:analyzer/dart/analysis/features.dart';
import 'package:analyzer/dart/ast/ast.dart';

var ast = parseFile('./lib/mistletoe.dart', featureSet: FeatureSet.fromEnableFlags([])).unit;
...

#Sample code

Viewing the tree

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/scanner.dart';
import 'dart:io';
//code to parse
String src = r"""
import 'package:mistletoe/mistletoe.dart';
var o = new Object();
main(){
  f(o);
}
f(p){
  var o = p;
  o = o as Object;
  print(o);
}
""";

main() async {
//  var src = await new File('./lib/mistletoe.dart').readAsString();
  var ast = parseCompilationUnit(src
    ,parseFunctionBodies: true);
  var nodes = flatten_tree(ast);
  var types ={};
  for(var n in nodes){
    types[n.runtimeType] ??= [];
    types[n.runtimeType].add(n);
  }
  var data = [];
  for(var k in types.keys){
    data.add(k.toString());
    for(var e in types[k]){
      data.add('\t'+e.toString());
    }
  }
  data = data.join('\n');
  print(data);
//  await new File('./lib/node_samples.txt').writeAsString(data);
}

List flatten_tree(AstNode n,[int depth=9999999]){
  var que = [];
  que.add(n);
  var nodes = [];
  int nodes_count = que.length;
  int dep = 0;
  int c = 0;
  if(depth == 0) return [n];
  while(que.isNotEmpty){
    var node = que.removeAt(0);
    if(node is! AstNode) continue;
    for(var cn in node.childEntities){
      nodes.add(cn);
      que.add(cn);
    }
    //Keeping track of how deep in the tree
    ++c;
    if(c == nodes_count){
      ++ dep; // One layer done
      if(depth <= dep) return nodes;
      c = 0;
      nodes_count = que.length;
    }
  }
  return nodes;
}
show(node){
  print('Type: ${node.runtimeType}, body: $node');
}

Editing node

d.on(o).hi = 'bye' to d.on(o).set("hi", 'bye')

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/error.dart';
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/generated/scanner.dart';
String src = """
  Dynamism d = new Dynamism(expert:true);
main(){
  var o = new Object();
  d.on(o).hi = 'bye';
}
""";
main(){
  var ast = parseCompilationUnit(src,parseFunctionBodies: true);
  print('initial value: ');
  print(ast.toSource());
  var v = new Visitor();
  ast.visitChildren(v);
  print('After modification:');
  print(ast.toSource());
}
class Visitor extends RecursiveAstVisitor{
  @override
  visitAssignmentExpression(AssignmentExpression node){
    //filter
    var p = new RegExp(r'.*\.on\(\w\)');
    if(!p.hasMatch(node.toString())) return;

    //replace
    SimpleStringLiteral ssl =
    _create_SimpleStringLiteral(node);
    node.parent.accept(new NodeReplacer(node,ssl));
  }
}

SimpleStringLiteral _create_SimpleStringLiteral(AstNode node){
  String new_string = modify(node.toString());
  int line_num = node.offset;
  //holds the position and type
  StringToken st = new StringToken(
      TokenType.STRING,new_string,line_num);
  return new SimpleStringLiteral(st, new_string);
}
String modify(String s){
  List parts = s.split('=');
  var value = parts[1];
  List l = parts[0].split('.');
  String dynamism = l.sublist(0,l.length-1).join('.');
  String propertyName = l.last.trim();
  return '${dynamism}.set("${propertyName}",${value})';
}

Changing Keyword: var o = new Object(); to Object o = new Object():

Note: editing A VariableDeclarationList is harder than editing other nodes for some reason.

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/scanner.dart';

//code to parse
String src = r"""
var d = new Dynamism(expert: true);
var o = new Object();
void main() {
  var e = new Object();
}
""";
main(){
  var result = var_to_object(src);
  print(result);
}
String var_to_object(String s){
  var ast = parseCompilationUnit(
      s,parseFunctionBodies: true);

// list imports
//  var directives = ast.directives;
//  print(directives);

  var v = new Visitor();
  ast.visitChildren(v);
  return ast.toSource();
}
class Visitor extends RecursiveAstVisitor{
  _is_type_Object(n){
    for( var c in n.childEntities){
      if(c is ConstructorName){
        if(c.toString() == 'Object')
          return true;
      }
      if(c is VariableDeclarationList)
        return _is_type_Object(c);
      if(c is VariableDeclaration)
        return _is_type_Object(c);
      if(c is InstanceCreationExpression)
        return _is_type_Object(c);
    }
    return false;
  }
  @override
  visitVariableDeclarationList(VariableDeclarationList n){
    //if n does not contain new Dynamism(expert:true) return;
    if(!_is_type_Object(n)) return;

//    Some note on the Keyword class
//     pseudo keywords are keywords that can be used as identifiers.
//     e.g. 
//     const Keyword("await", isPseudo: true),
//     const Keyword("yield", isPseudo: true)];

// syntax arguments samples
//    static const Keyword LIBRARY = const Keyword._('LIBRARY', "library", true);
//    static const Keyword NEW = const Keyword._('NEW', "new");
//    static const Keyword NULL = const Keyword._('NULL', "null");
//    static const Keyword OPERATOR = const Keyword._('OPERATOR', "operator", true);

    // name and syntax seems to have been switched around
    var kw = const Keyword('class','Object', false);
    var kt = new KeywordToken(kw,n.keyword.offset);
    var ndl = new VariableDeclarationList(
        n.documentationComment,[],kt,
        n.type,n.variables);
    //changing var d to Object d
    n.parent.accept(new NodeReplacer(n,ndl));
  }
}

Find the nearest declaration/assignment/FormalParameter/TypeName of a given variable

Note: The code probably contains bugs still. I should reorganize the code, but I'm a little short on time now and leaving as I prototyped.

import 'package:analyzer/analyzer.dart';
import 'package:analyzer/src/generated/ast.dart';
import 'package:analyzer/src/generated/scanner.dart';


String src = r"""
final String a = 'a';
class A{
  static Dynamism d = new Dynamism(expert:true);
  A(a){
    a = 'changed';
    print(a);
    var o = new Object();
    o = a as String;
    print(o);
  }
}""";

main() async{
  var ast = parseCompilationUnit(
      src,parseFunctionBodies: true);
  //Get all SimpleIdentifier nodes in ast
  var simple_nodes = new List<AstNode>();
  for(var n in flatten_tree(ast)){
    if(n is SimpleIdentifier){
      simple_nodes.add(n);
    }
  }
  for(SimpleIdentifier n in simple_nodes){
    var a = guess_effective_assignment_of(n);
    var d = get_declaration_of(n);
    var f = get_formal_parameter_of(n);
    if(a == null && d == null && f == null)
      continue;
    print('------------');
    print('For $n at: ${n.offset} in the node:'
        '${n.parent.parent}');
    print('The effective definition is:');
    var definitions = guess_effective_definition_of(n);
    var first_candidate = definitions.first;
    var second_candidate = definitions.second;
    var third_candidate = definitions.third;
    print('\t`${first_candidate}` of type '
        '${first_candidate.runtimeType}');
    print('\tSecond candidate is `${second_candidate}`');
    print('\tThird candidate is `${third_candidate}`');
    print('Where the enclosing block is:');
    print('\t${get_surrounding_block(n)}');
    var type = get_type_name(n);
    print('type or class name:$type');
  }

}
AstNode get_surrounding_block(node){
  if(node == null) return null;
  while(true){
    if(node.parent == null) return null;
    node = node.parent;
    if(is_scope(node))
      return node;

  }
}
String get_type_name(SimpleIdentifier node){
  Definitions definitions =
    guess_effective_definition_of(node);
  var g = definitions.first;
  //Takes only SimpleFormalParameter or Declaration
  TypeName _extract_TypeName_from(d){
    if(d == null) return null;
    for(var t in d.childEntities)
      if(t is TypeName) return t;
  }
  var name;
  if(g is VariableDeclaration){
    name = _extract_TypeName_from(g);
    if(name == null || name == 'var'){
      name = extract_constructor_name_or_rvalue(g);
      //Not supporting MethodInvocation
      if(name is MethodInvocation)
        return null;

      //rvalue is a variable. calling self.
      if(name is SimpleIdentifier)
        return get_type_name(name);
    }
  }
  if(g is AssignmentExpression){
    // checking FormalParameter or
    // VariableDeclaration for type
    var closer = definitions.second;
    var type_name = _extract_TypeName_from(closer);
    if(type_name != 'var' && type_name != null)
      return type_name.toString();
    //TypeName is var

    for(var e in g.childEntities){
      if(e is AsExpression){
        return e.childEntities.last.toString();
      }
    }
    name = extract_constructor_name_or_rvalue(g);
    //Not supporting MethodInvocation
    if(name is MethodInvocation)
      return null;

    //rvalue is a variable. calling self.
    if(name is SimpleIdentifier)
      return get_type_name(name);
    //A literal is handled later
  }
  if(g is FormalParameter){
    if(g.childEntities.length>1){
      name = _extract_TypeName_from(g);
    }else{
      name = 'var';
    }
  }

  if(name is Literal){
    name = name.runtimeType.toString();
    name = name.replaceAll('Simple','')
        .replaceAll('Literal','');
  }else{
    name = name.toString();
  }
  return name;
}


///finds the variable declaration for the given node
///Takes SimpleIdentifier representing a variable
List get_declaration_of(var n){
  // Check if n is part of VariableDeclaration
  // and is the variable being defined.
  var r = new List(2);
  var original = n;
  while(n.parent !=null){
    if(n is VariableDeclaration)
      if(n.childEntities.first == original){
        r[0]=n;r[1]=0;
        return r;
      }else{
        //n is not the variable being defined
        break;
      }
    n = n.parent;
  }
  n = original;

  // The variable's identifier is defined up the lines
  // or in outer scope.

  //searching local scope
  var block = get_surrounding_block(n);
  var declarations = extract_scope_wide_declarations(block);
  for(var d in declarations) {
    if(d.offset > n.offset) break;
    var v = d?.childEntities?.first;
    if(v == original){
      r[0]=d;r[1]=0;
      return r;}
  }

  //searching the outer scopes
  block = get_surrounding_block(block);
  declarations.clear();
  int count = -1;
  while(block != null){
    var decls = extract_scope_wide_declarations(block);
    for(var d in decls) {
      var v = d.childEntities.first;
      if(v.toString() == original.toString()){
        r[0]=d;r[1]=count;
        return r;
      }
    }
    block = get_surrounding_block(block);
    --count;
  }
  return null;//could not find the declaration
}
///Fetch declarations that have effect in
///all the children blocks of the given node.
///Does not cover FormalParameterList.
///Does not enter a node that is a block.
List extract_scope_wide_declarations(AstNode node){
  var nodes = [];
  var declarations = [];
  nodes.addAll(node.childEntities);

  //pushing more nodes and skipping blocks
  while(nodes.isNotEmpty){
    var e = nodes.removeAt(0);
    if(e is! AstNode) continue;
    if(e is VariableDeclaration){
      declarations.add(e);
      continue;
    }
    if(!is_scope(e) && e is AstNode){
        nodes.addAll(e.childEntities);
    }
  }
  return declarations;
}
///Returns true if the given node has its
///scope.
///Returns also true for:
/// MethodDeclaration
/// FunctionDeclaration
/// CompilationUnit
/// ClassDeclaration
///
///as they have a scope.
bool is_scope(node){
    if(node is Block||
        node is CompilationUnit ||
        node is Block||
        //below are for extracting arguments
        node is MethodDeclaration ||
        node is FunctionDeclaration ||
        node is ConstructorDeclaration||
        //FieldDeclarations
        node is ClassDeclaration ||
        //Block should cover these but jus in case
        node is IfStatement ||
        node is WhileStatement||
        node is DoStatement ||
        node is TryStatement ||
        node is SwitchStatement) return true;
  return false;
}

/// Takes a VariableDeclaration node.
/// Returns a ConstructorName node or
/// rvalue; includes MethodInvocation.
/// todo write test
extract_constructor_name_or_rvalue(n){
  var nodes = flatten_tree(n,1);
  for(var node in nodes){
    if(node is Literal) return node;
    if(node is TypeName){
      return node;
    }
    if(node is InstanceCreationExpression)
      for(var e in node.childEntities)
        if(e is ConstructorName) return e;
    if(node is MethodInvocation){
      return node;
    }
  }
  //rvalue is a variable
  return nodes.last;
}


/// Returns the closest definition of n.
/// sub blocks are ignored.
///
///  A guess because it would fail
/// if a conditional modifies the value
/// of the variable in runtime.
///
/// e.g.
///
///     var a = 'hi';
///     if(user_input){
///       a = new Object();
///     }
///     //a is an Object not String
///
/// also does not look into constructor.
///
/// todo write test
Definitions guess_effective_definition_of(n){
  List al = guess_effective_assignment_of(n);
  List dl = get_declaration_of(n);
  List fl = get_formal_parameter_of(n);

  if(al == null && dl == null && fl == null )
    return null;

  if(al == null && fl == null && dl != null)
    return new Definitions()
      ..first = dl[0]
      ..first_relative_depth=dl[1]
      ..length = 1;

  if(dl == null && fl == null && al != null)
    return new Definitions()
      ..first = al[0]
      ..first_relative_depth = al[1]
      ..length = 1;

  if(fl != null && al == null && dl == null)
    return new Definitions()
      ..first = fl[0]
      ..first_relative_depth = fl[1]
      ..length = 1;;

  Definitions r = new Definitions();
  // Guessing which declaration or
  // definition takes precedence.
  var l = []..add(dl)..add(al)..add(fl);
  l.removeWhere((e)=>e==null);

  // 0 means local scope.
  // -1 means one scope up.
  //Comparator returns negative if a comes before b,
  //0 if equal, positive if a comes after b.
  l.sort((a,b)=> b[1] - a[1]);
  // if AssignmentExpression and
  // VariableDeclaration are
  // in the same scope,
  // AssignmentExpression always
  // takes precedence.
  l.sort((a,b)=>
    a == al && b == dl && al[1] == dl[1] ?
      -1:0
  );
  //maybe I should just return a list?
  try {
    r.length = l.length;
    r.first = l[0][0];
    r.first_relative_depth = l[0][1];
    r.second = l[1][0];
    r.second_relative_depth = l[1][1];
    r.third = l[2][0];
    r.third_relative_depth = l[2][1];
  }on RangeError catch(e){
    // do nothing
  }
  return r;
}
/// Searches for an AssignmentExpression
/// that is most likely to define the value
/// of the variable denoted by the identifier
/// in n.
///
///  A guess because it would fail
/// if a conditional modifies the value
/// of the variable on runtime.
///
/// e.g.
///
///     var a = 'hi';
///     if(user_input){
///       a = new Object();
///     }
///     //a is an Object not String
///     //but this function does not
///     //know that.
///
guess_effective_assignment_of(SimpleIdentifier n){
  //checks if n is part of a variable declaration
  //Code is mostly duplicate of get_declaration_of
  // AssignmentExpression does not contain
  // VariableDeclaration or vice versa.
  var r = new List(2);
  var node = n;
  while(node.parent !=null){
    if(node is AssignmentExpression)
      if(node.childEntities.first == n){
        r[0]=node;r[1]=0;
        return r;
      }else{
        //n is not the variable being defined
        break;
      }
    node = node.parent;
  }

  //Search local scope
  var b = get_surrounding_block(n);
  AstNode closest;
  for(var node in extract_assignments_to_n_from(n, b)){
    if(n.offset > node.offset){
      closest = node;
    }else{break;}
  }
  if(closest != null){
    r[0]=closest;r[1]=0;
    return r;
  }
  //outer scopes
  b = get_surrounding_block(b);
  int count = -1;
  while(b != null){
    for(var a in extract_assignments_to_n_from(n,b)) {
      var v = a?.childEntities?.first;
      if(v.toString() == n.toString()){
        r[0]=a;r[1]=count;
        return r;
      }
    }
    b = get_surrounding_block(b);
    --count;
  }
  return null;
}

///Searches scopes upward for an
///AssignmentExpression for the
///variable denoted by the identifier
///in n.
///
/// Returns a list of AssignmentExpression.
/// todo test this
extract_assignments_to_n_from(
    SimpleIdentifier n,
    AstNode in_scope,
    [int search_depth=2]){
  var nodes = flatten_tree(in_scope,search_depth)
      .where((e)=>e is AssignmentExpression);
  var r = [];
  for(AstNode node in nodes){
    String i = node.childEntities.first.toString();
    if(i == n.toString()){
      r.add(node);
    }
  }
  return r;
}
///Takes any node.
///
///Searches scopes upward to find
///the closest FormalParameterList
///
/// Returns a list of
///   1.  FormalParameterList
///   2.  Its depth relative to n
///
/// Or null.
///
List get_nearest_formal_parameter_list(n){
  var r = new List(2);
  int count = 0;
  while(true){
    if(n is FunctionDeclaration ||
        n is MethodDeclaration ||
        n is ConstructorDeclaration)
      break;
    n = get_surrounding_block(n);
    --count;
    if(n == null) return n;
  }
  for(var e in n.childEntities){
    if(e is FormalParameterList){
      r[0] = e;r[1] = count;
      return r;
    }
  }
  //empty FormalParameterList
  return null;
}
///Searches up for a FormalParameterList.
///
///Returns a list of:
///
///  1. FormalParameter matching
///  the identifier of n when stringified
///  if such exists. Returns null
///  otherwise.
///
///   2.  The number of scopes moved up
///   from the scope n belongs to.
///
get_formal_parameter_of(n){
  var r = new List(2);

  //returns [FormalParameterList, int]
  var fl = get_nearest_formal_parameter_list(n);

  if( fl == null) return null;
  List names = extract_arg_names(fl[0]);
  for(var name in names){
    if(name.toString() == n.toString()){
      r[0]=name.parent;
      r[1]=fl[1];
      return r;
    }
  }
  return null;
}
///Takes:
///FunctionDeclaration
///FunctionExpression
///FormalParameterList
///
///Returns:
///Positional argument names :e.g. a and b in `f(a,b){return a+b;}`
///Named option names:e.g. your_name in `f({String your_name}){...}`
///
List<SimpleIdentifier> extract_arg_names(AstNode n){
  var fpl;
  if(n is! FormalParameterList){
    for(var e in flatten_tree(n)){
      if(e is FormalParameterList){
        fpl = e;
        break;
      }
    }
  }else if(n is FormalParameterList){
    fpl = n;
  }
  var r = [];
  for(var c in fpl.childEntities){
    if(c is! FormalParameter)
      continue;
    for(var cc in c.childEntities){
      //filtering String int etc
      if(cc is! TypeName) r.add(cc);
    }
  }
  return r;
}
List flatten_tree(AstNode n,[int depth=9999999]){
  var que = [];
  que.add(n);
  var nodes = [];
  int nodes_count = que.length;
  int dep = 0;
  int c = 0;
  if(depth == 0) return [n];
  while(que.isNotEmpty){
    var node = que.removeAt(0);
    if(node is! AstNode) continue;
    for(var cn in node.childEntities){
      nodes.add(cn);
      que.add(cn);
    }
    //Keeping track of how deep in the tree
    ++c;
    if(c == nodes_count){
      ++ dep; // One layer done
      if(depth <= dep) return nodes;
      c = 0;
      nodes_count = que.length;
    }
  }
  return nodes;
}
show(node){
  print('Type: ${node.runtimeType}, body: $node');
}
//todo test this: could not tokenize a node completely
class FlattenVisitor extends BreadthFirstVisitor{
  List _nodes;
  FlattenVisitor(this._nodes):super();
  @override
  visitNode(AstNode n){
    _nodes.add(n);
  }
}

/// Returned by guess_effective_definition_of
///
/// Confusing but a Definitions instance may
/// include `var a;`; a declaration but also
/// defining the type of a as dynamic.
///
/// Having moved upward to find the definition/
/// declaration results in a negative depth.
/// If definition/declaration is found locally,
/// depth is set to 0.
///
class Definitions{
  AstNode first;
  int first_relative_depth;
  AstNode second;
  int second_relative_depth;
  AstNode third;
  int third_relative_depth;
  int length;
}

Analyzer code is still work in progress. I guess the rework of the Dart code auto-generated from the early Java source isn't done yet. They are working on improving the public API and then they will work on better docs as well.

Without having a closer look myself yet I think these packages should make use of this as well

• https://github.com/dart-lang/linter
• https://github.com/dart-lang/dart_style
• https://github.com/dart-lang/source_gen
• https://github.com/dart-lang/dartdoc

dart 2.11, from test code

import 'package:analyzer/dart/analysis/utilities.dart';
import 'package:analyzer/dart/analysis/results.dart';
import 'package:test/test.dart';

void main() {
  test('hello_parser', () {
    String content = '''
void main() => print('Hello, world!')
''';
    ParseStringResult result = parseString(content: content, throwIfDiagnostics: false);
    // print(result.unit.toSource());
    expect(result.content, content);
    expect(result.errors, hasLength(1));
    expect(result.lineInfo, isNotNull);
    expect(result.unit.toString(), equals("void main() => print('Hello, world!');"));
  });
}