Most efficient way to insert Rows into MySQL Database
Here is my "multiple inserts"-code.
The insertion of 100k rows took instead of 40 seconds only 3 seconds!!
public static void BulkToMySQL()
{
string ConnectionString = "server=192.168.1xxx";
StringBuilder sCommand = new StringBuilder("INSERT INTO User (FirstName, LastName) VALUES ");
using (MySqlConnection mConnection = new MySqlConnection(ConnectionString))
{
List<string> Rows = new List<string>();
for (int i = 0; i < 100000; i++)
{
Rows.Add(string.Format("('{0}','{1}')", MySqlHelper.EscapeString("test"), MySqlHelper.EscapeString("test")));
}
sCommand.Append(string.Join(",", Rows));
sCommand.Append(";");
mConnection.Open();
using (MySqlCommand myCmd = new MySqlCommand(sCommand.ToString(), mConnection))
{
myCmd.CommandType = CommandType.Text;
myCmd.ExecuteNonQuery();
}
}
}
The created SQL-statement looks like this:
INSERT INTO User (FirstName, LastName) VALUES ('test','test'),('test','test'),... ;
Update: Thanks Salman A I added MySQLHelper.EscapeString
to avoid code injection which is internally used when you use parameters.
I did a small test using three things MySqlDataAdapter,transactions and UpdateBatchSize. It is about 30 times faster than your first example. Mysql is running on separate box so there is latency involved. The batchsize might need some tuning. Code follows:
string ConnectionString = "server=xxx;Uid=xxx;Pwd=xxx;Database=xxx";
string Command = "INSERT INTO User2 (FirstName, LastName ) VALUES (@FirstName, @LastName);";
using (var mConnection = new MySqlConnection(ConnectionString))
{
mConnection.Open();
MySqlTransaction transaction = mConnection.BeginTransaction();
//Obtain a dataset, obviously a "select *" is not the best way...
var mySqlDataAdapterSelect = new MySqlDataAdapter("select * from User2", mConnection);
var ds = new DataSet();
mySqlDataAdapterSelect.Fill(ds, "User2");
var mySqlDataAdapter = new MySqlDataAdapter();
mySqlDataAdapter.InsertCommand = new MySqlCommand(Command, mConnection);
mySqlDataAdapter.InsertCommand.Parameters.Add("@FirstName", MySqlDbType.VarChar, 32, "FirstName");
mySqlDataAdapter.InsertCommand.Parameters.Add("@LastName", MySqlDbType.VarChar, 32, "LastName");
mySqlDataAdapter.InsertCommand.UpdatedRowSource = UpdateRowSource.None;
var stopwatch = new Stopwatch();
stopwatch.Start();
for (int i = 0; i < 50000; i++)
{
DataRow row = ds.Tables["User2"].NewRow();
row["FirstName"] = "1234";
row["LastName"] = "1234";
ds.Tables["User2"].Rows.Add(row);
}
mySqlDataAdapter.UpdateBatchSize = 100;
mySqlDataAdapter.Update(ds, "User2");
transaction.Commit();
stopwatch.Stop();
Debug.WriteLine(" inserts took " + stopwatch.ElapsedMilliseconds + "ms");
}
}