How to validate uploaded file in ASP.Net Core

This is how I use the custom validation attributes, almost the same as @xueli-chen 's answer but production-ready.

FileExtensionsAttribute

using System;
using System.ComponentModel.DataAnnotations;
using System.Globalization;
using System.IO;
using System.Linq;

using Microsoft.AspNetCore.Http;

using NewsPassWebApi.Properties;

namespace NewsPassWebApi.Models.DataAnnotaions
{
    /// <summary>
    /// Validation attribute to assert an <see cref="IFormFile">IFormFile</see> property, field or parameter has a specific extention.
    /// </summary>
    [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter, AllowMultiple = false)]
    public sealed class FileExtensionsAttribute : ValidationAttribute
    {
        private string _extensions;

        /// <summary>
        /// Gets or sets the acceptable extensions of the file.
        /// </summary>
        public string Extensions
        {
            get
            {
                // Default file extensions match those from jquery validate.
                return string.IsNullOrEmpty(_extensions) ? "png,jpg,jpeg,gif" : _extensions;
            }
            set
            {
                _extensions = value;
            }
        }

        private string ExtensionsNormalized
        {
            get
            {
                return Extensions.Replace(" ", "", StringComparison.Ordinal).ToUpperInvariant();
            }
        }

        /// <summary>
        /// Parameterless constructor.
        /// </summary>
        public FileExtensionsAttribute() : base(() => Resources.FileExtensionsAttribute_ValidationError)
        { }

        /// <summary>
        /// Override of <see cref="ValidationAttribute.IsValid(object)"/>
        /// </summary>
        /// <remarks>
        /// This method returns <c>true</c> if the <paramref name="value"/> is null.  
        /// It is assumed the <see cref="RequiredAttribute"/> is used if the value may not be null.
        /// </remarks>
        /// <param name="value">The value to test.</param>
        /// <returns><c>true</c> if the value is null or it's extension is not invluded in the set extensionss</returns>
        public override bool IsValid(object value)
        {
            // Automatically pass if value is null. RequiredAttribute should be used to assert a value is not null.
            if (value == null)
            {
                return true;
            }

            // We expect a cast exception if the passed value was not an IFormFile.
            return ExtensionsNormalized.Split(",").Contains(Path.GetExtension(((IFormFile)value).FileName).ToUpperInvariant());
        }

        /// <summary>
        /// Override of <see cref="ValidationAttribute.FormatErrorMessage"/>
        /// </summary>
        /// <param name="name">The name to include in the formatted string</param>
        /// <returns>A localized string to describe the acceptable extensions</returns>
        public override string FormatErrorMessage(string name)
        {
            return string.Format(CultureInfo.CurrentCulture, ErrorMessageString, name, Extensions);
        }
    }
}

FileSizeAttribute

using System;
using System.ComponentModel.DataAnnotations;
using System.Globalization;

using Microsoft.AspNetCore.Http;

using NewsPassWebApi.Properties;

namespace NewsPassWebApi.Models.DataAnnotaions
{
    /// <summary>
    /// Validation attribute to assert an <see cref="IFormFile">IFormFile</see> property, field or parameter does not exceed a maximum size.
    /// </summary>
    [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field | AttributeTargets.Parameter, AllowMultiple = false)]
    public sealed class FileSizeAttribute : ValidationAttribute
    {
        /// <summary>
        /// Gets the maximum acceptable size of the file.
        /// </summary>
        public long MaximumSize { get; private set; }

        /// <summary>
        /// Gets or sets the minimum acceptable size of the file.
        /// </summary>
        public int MinimumSize { get; set; }

        /// <summary>
        /// Constructor that accepts the maximum size of the file.
        /// </summary>
        /// <param name="maximumSize">The maximum size, inclusive.  It may not be negative.</param>
        public FileSizeAttribute(int maximumSize) : base(() => Resources.FileSizeAttribute_ValidationError)
        {
            MaximumSize = maximumSize;
        }

        /// <summary>
        /// Override of <see cref="ValidationAttribute.IsValid(object)"/>
        /// </summary>
        /// <remarks>
        /// This method returns <c>true</c> if the <paramref name="value"/> is null.  
        /// It is assumed the <see cref="RequiredAttribute"/> is used if the value may not be null.
        /// </remarks>
        /// <param name="value">The value to test.</param>
        /// <returns><c>true</c> if the value is null or it's size is less than or equal to the set maximum size</returns>
        /// <exception cref="InvalidOperationException"> is thrown if the current attribute is ill-formed.</exception>
        public override bool IsValid(object value)
        {
            // Check the lengths for legality
            EnsureLegalSizes();

            // Automatically pass if value is null. RequiredAttribute should be used to assert a value is not null.
            // We expect a cast exception if the passed value was not an IFormFile.
            var length = value == null ? 0 : ((IFormFile)value).Length;

            return value == null || (length >= MinimumSize && length <= MaximumSize);
        }

        /// <summary>
        /// Override of <see cref="ValidationAttribute.FormatErrorMessage"/>
        /// </summary>
        /// <param name="name">The name to include in the formatted string</param>
        /// <returns>A localized string to describe the maximum acceptable size</returns>
        /// <exception cref="InvalidOperationException"> is thrown if the current attribute is ill-formed.</exception>
        public override string FormatErrorMessage(string name)
        {
            EnsureLegalSizes();

            string errorMessage = MinimumSize != 0 ? Resources.FileSizeAttribute_ValidationErrorIncludingMinimum : ErrorMessageString;

            // it's ok to pass in the minLength even for the error message without a {2} param since String.Format will just ignore extra arguments
            return string.Format(CultureInfo.CurrentCulture, errorMessage, name, MaximumSize, MinimumSize);
        }

        /// <summary>
        /// Checks that MinimumSize and MaximumSize have legal values.  Throws InvalidOperationException if not.
        /// </summary>
        private void EnsureLegalSizes()
        {
            if (MaximumSize < 0)
            {
                throw new InvalidOperationException(Resources.FileSizeAttribute_InvalidMaxSize);
            }

            if (MaximumSize < MinimumSize)
            {
                throw new InvalidOperationException(string.Format(CultureInfo.CurrentCulture, Resources.RangeAttribute_MinGreaterThanMax, MaximumSize, MinimumSize));
            }
        }
    }
}

now you can use those like any built-in validation attribute including custom/localized error messages, max-min file sizes, and file extensions.

Following previous comment, you can add this class:

public class ValidateModelStateFilter : ActionFilterAttribute
{
    public override void OnActionExecuting(ActionExecutingContext context)
    {
        if (context.ModelState.IsValid)
        {
            return;
        }

        var validationErrors = context.ModelState
            .Keys
            .SelectMany(k => context.ModelState[k].Errors)
            .Select(e => e.ErrorMessage)
            .ToArray();

        var json = new JsonErrorResponse
        {
            Messages = validationErrors
        };

        context.Result = new BadRequestObjectResult(json);
    }
}

Finally, add the filter to the controllers in Startup class:

 services.AddControllers(options =>
            {
                options.Filters.Add(typeof(ValidateModelStateFilter));
            })

You could custom validation attribute MaxFileSizeAttribute like below

MaxFileSizeAttribute

public class MaxFileSizeAttribute : ValidationAttribute
{
    private readonly int _maxFileSize;
    public MaxFileSizeAttribute(int maxFileSize)
    {
        _maxFileSize = maxFileSize;
    }

    protected override ValidationResult IsValid(
    object value, ValidationContext validationContext)
    {
        var file = value as IFormFile;
        if (file != null)
        {
           if (file.Length > _maxFileSize)
            {
                return new ValidationResult(GetErrorMessage());
            }
        }

        return ValidationResult.Success;
    }

    public string GetErrorMessage()
    {
        return $"Maximum allowed file size is { _maxFileSize} bytes.";
    }
}

AllowedExtensionsAttribute

public class AllowedExtensionsAttribute : ValidationAttribute
{
    private readonly string[] _extensions;
    public AllowedExtensionsAttribute(string[] extensions)
    {
        _extensions = extensions;
    }
    
    protected override ValidationResult IsValid(
    object value, ValidationContext validationContext)
    {
        var file = value as IFormFile;
        if (file != null)
        {
            var extension = Path.GetExtension(file.FileName);
            if (!_extensions.Contains(extension.ToLower()))
            {
                return new ValidationResult(GetErrorMessage());
            }
        }
        
        return ValidationResult.Success;
    }

    public string GetErrorMessage()
    {
        return $"This photo extension is not allowed!";
    }
}

Add MaxFileSize attribute and AllowedExtensions attribute to Photo property

public class UserViewModel
{
        [Required(ErrorMessage = "Please select a file.")]
        [DataType(DataType.Upload)]
        [MaxFileSize(5* 1024 * 1024)]
        [AllowedExtensions(new string[] { ".jpg", ".png" })]
        public IFormFile Photo { get; set; }
 }

You can implement IValidatableObject to validate your model.

public class UserViewModel : IValidatableObject
    {
        [Required(ErrorMessage = "Please select a file.")]
        [DataType(DataType.Upload)]
        public IFormFile Photo { get; set; }

        public IEnumerable<ValidationResult> Validate(ValidationContext validationContext)
        {
            var photo = ((UserViewModel)validationContext.ObjectInstance).Photo;
            var extension = Path.GetExtension(photo.FileName);
            var size = photo.Length;

            if (!extension.ToLower().Equals(".jpg"))
                yield return new ValidationResult("File extension is not valid.");

           if(size > (5 * 1024 * 1024))
                yield return new ValidationResult("File size is bigger than 5MB.");
        }
    }