SQLite insert :memory: performance

I'm looking for a "cache" solution using in-memory tables with SQLite. Fast insert, and fast selects are high priority.

The problem is that I do not get the insert speed I was expecting in SQLite.

I'm benchmarking SQLite in-memory tables vs. doing BulkCopyInserts into a SQL Server database. I can BulkCopy 100 000 records with a speed of 50 000 records /sec into a table, but using SQLite I have a speed of 12 500 rec/sec.

Any idea what I can do to improve performance? Is there no way to push the data into the table without looping and doing insert for each record?

/Erik

        private void SQLLiteRunButton_Click(object sender, EventArgs e)
    {

        int noOfRecords = Convert.ToInt32(NoOfRecordsTextBox.Text);

        SQLLiteListBox.Items.Add("Start generating "+noOfRecords+" in memory");
        DataTable table = GetTable(noOfRecords);

        var dbConnection = new SQLiteConnection("Data Source=:memory:;Version = 3;");

        dbConnection.Open();

        //Create Target Table
        string sql = "CREATE TABLE LedgerSumTemp([RowID] [int] NOT NULL,[Client_ID] [nvarchar](50) NULL,[ISOMONTH] [nvarchar](50) NULL,[Date] [datetime] NULL, [Dim1] [nvarchar](50) NULL,[Dim2] [nvarchar](50) NULL,   [Dim3] [nvarchar](50) NULL, [Dim4] [nvarchar](50) NULL, [Amount] [decimal](18, 6) NULL)";

        SQLiteCommand command = new SQLiteCommand(sql, dbConnection);
        command.ExecuteNonQuery();

        //Set insertTimeStart
        Stopwatch stopwatch = new Stopwatch();
        stopwatch.Start();

        //Fill table with data fast - Alternative 2 – using params
        using (var cmd = new SQLiteCommand(dbConnection))
        {


            using (var transaction = dbConnection.BeginTransaction())
            {

                cmd.CommandText = "INSERT INTO LedgerSumTemp (RowID, Client_ID, ISOMONTH, Date, Dim1, Dim2, Dim3, Dim4, Amount) VALUES (@RowID,@Client_ID,@ISOMONTH,@Date,@Dim1,@Dim2,@Dim3,@Dim4,@Amount);";
                cmd.CommandType = CommandType.Text;

                foreach (DataRow row in table.Rows)
                {
                    cmd.Parameters.Add(new SQLiteParameter("@RowID", row["RowID"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Client_ID", row["Client_ID"]));
                    cmd.Parameters.Add(new SQLiteParameter("@ISOMONTH", row["ISOMONTH"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Date", row["Date"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Dim1", row["Dim1"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Dim2", row["Dim2"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Dim3", row["Dim3"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Dim4", row["Dim4"]));
                    cmd.Parameters.Add(new SQLiteParameter("@Amount", row["Amount"]));

                    cmd.ExecuteNonQuery();

                }

                transaction.Commit();
            }
        }



        stopwatch.Start();
        int mS = Convert.ToInt32(stopwatch.Elapsed.TotalMilliseconds);

        double s = mS / 1000;
        double recPerSec = (noOfRecords / s);

        SQLLiteListBox.Items.Add(noOfRecords.ToString()+" Time: " + mS.ToString()+ " mSec"+" Speed: "+recPerSec.ToString() +" RecPerSec");
        SQLLiteListBox.Items.Add("--------END-------");

}

    static DataTable GetTable(int NoOfRows)
    {

        int noOfRows = NoOfRows;

        //Create DataTable in memory
        DataTable table = new DataTable();
        table.TableName = "LedgerSumTemp";
        table.Columns.Add("RowID", typeof(int));
        table.Columns.Add("Client_ID", typeof(string));
        table.Columns.Add("ISOMONTH", typeof(string));
        table.Columns.Add("Date", typeof(DateTime));
        table.Columns.Add("Dim1", typeof(string));
        table.Columns.Add("Dim2", typeof(string));
        table.Columns.Add("Dim3", typeof(string));
        table.Columns.Add("Dim4", typeof(string));
        table.Columns.Add("Amount", typeof(double));

        //Add rows
        for (int i = 0; i < noOfRows; i++)
        {
            table.Rows.Add(i, "10", "2013"+i.ToString(), DateTime.Now, "3010", "110", "A58", "1000", Convert.ToDouble(i));
        }

        return table;
    }

---

Your code is creating new parameter objects in each loop iteration.

You should reuse them and just change their values; something like this:

...
var paramRowID = cmd.CreateParameter();
paramRowID.ParameterName = "@RowID";
cmd.Parameters.Add(paramRowID);
// same for the other parameters

foreach ...
{
    paramRowID.Value = row["RowID"];
    // same for the other parameters
    cmd.ExecuteNonQuery();
}

However, SQLite has no mechanism similar to BulkCopy; you always have to insert records one by one.

链接地址: http://www.djcxy.com/p/19816.html

上一篇: SQLite更新一个记录是非常（相对）慢

下一篇: SQLite插入：内存：性能