Can I use a Tag Helper in a custom Tag Helper that returns html?

No you cannot. TagHelpers are a Razor parse time feature.

One alternative is creating a TagHelper and manually invoking its ProcessAsync/Process method. Aka:

var anchorTagHelper = new AnchorTagHelper
{
    Action = "Home",
};
var anchorOutput = new TagHelperOutput("a", new TagHelperAttributeList(), (useCachedResult, encoder) => new HtmlString());
var anchorContext = new TagHelperContext(
    new TagHelperAttributeList(new[] { new TagHelperAttribute("asp-action", new HtmlString("Home")) }),
    new Dictionary<object, object>(),
    Guid.NewGuid());
await anchorTagHelper.ProcessAsync(anchorContext, anchorOutput);
output.Content.SetHtmlContent(anchorOutput);

If anyone's looking to reuse the built-in tag helpers from asp.net core, you can use the IHtmlGenerator instead. For reusing other types of tag helpers, I haven't found a simpler option then @N. Taylor Mullen answer

Here is how to reuse the asp-action tag helper:

[HtmlTargetElement("helplink")]
public class RazorTagHelper : TagHelper
{
    private readonly IHtmlGenerator _htmlGenerator;

    public RazorTagHelper(IHtmlGenerator htmlGenerator)
    {
        _htmlGenerator = htmlGenerator;
    }

    [ViewContext]
    public ViewContext ViewContext { set; get; }

    public override void Process(TagHelperContext context, TagHelperOutput output)
    {
        output.TagName = "div";
        output.TagMode = TagMode.StartTagAndEndTag;
        var actionAnchor = _htmlGenerator.GenerateActionLink(
            ViewContext,
            linkText: "Home",
            actionName: "Index",
            controllerName: null,
            fragment: null,
            hostname: null,
            htmlAttributes: null,
            protocol: null,
            routeValues: null
            );
        var builder = new HtmlContentBuilder();
        builder.AppendHtml("Here's the link: ");
        builder.AppendHtml(actionAnchor);
        output.Content.SetHtmlContent(builder);
    }
}