Qt之Sqlite3 Insert优化

  最近在项目遇到一个问题，就是插入大量数据时启动程序会特别卡，一开始我使用线程缓解了一点，但是仍然会卡很久，今天在复习Sqlite相关内容时，突然发现了一个与我很相似的情况。究其主要原因还是执行INSERT语句时还有很大问题。在Sqlite官网关于INSERT的方式中INSERT过程中最后如果使用VALUES那条路会比select-stmt和DEFAULT -VALUES多走很多路，可能这样就导致了INSERT性能差异吧，实际测下来确实挺大的，当然也主要是old fraction那块太多的INSERT了，精简后只有5条INSERT语句，最终的结果就是old fraction执行会超过10S，而精简后只需要不到1S。

以下几个是相关的链接：

https://www.jianshu.com/p/faa5e852b76b

https://stackoverflow.com/questions/1609637/is-it-possible-to-insert-multiple-rows-at-a-time-in-an-sqlite-database/1734067

https://www.sqlite.org/lang_insert.html

/**
*    old fraction
*/
bool SqlInit::InitSql()
{
    QSqlQuery query(DBThreadConn);
    bool bSuccessLogCountTable = query.exec("create table LogCountTable (id INTEGER PRIMARY KEY, Type varchar(10), ucIndex varchar(10), LogCount varchar(10))");
    if(bSuccessLogCountTable)
    {
        uint8_t ucTemp = 1U;
        for (uint8_t i = ucTemp; i < ucTemp + COMMON::EBC_NUMBER; i++)
        {
            QString strtemp = "EB";
            QString str = "insert into LogCountTable(Type, ucIndex, LogCount) values('" + strtemp + "','" + QString::number(i) + "','" + QString::number(0) + "')";
            query.exec(str);
        }

        ucTemp += COMMON::EBC_NUMBER;

        for (uint8_t i = ucTemp; i < ucTemp + COMMON::EBC_NUMBER; i++)
        {
            QString strtemp = "FSB";
            QString str = "insert into LogCountTable(Type, ucIndex, LogCount) values('" + strtemp + "','" + QString::number(i - COMMON::EBC_NUMBER) + "','" + QString::number(0) + "')";
            query.exec(str);
        }
        ucTemp += COMMON::EBC_NUMBER;
        for (uint8_t i = ucTemp; i < ucTemp + COMMON::WC_MAX_NUM; i++)
        {
            QString strtemp = "Warning";
            QString str = "insert into LogCountTable(Type, ucIndex, LogCount) values('" + strtemp + "','" + QString::number(i - COMMON::EBC_NUMBER - COMMON::EBC_NUMBER) + "','" + QString::number(0) + "')";
            query.exec(str);
        }
        ucTemp += COMMON::WC_MAX_NUM;

        for (uint8_t i = ucTemp; i < ucTemp + COMMON::CC_FAULT_MAX; i++)
        {
            QString strtemp = "CC";
            QString str = "insert into LogCountTable(Type, ucIndex, LogCount) values('" + strtemp + "','" + QString::number(i - COMMON::EBC_NUMBER - COMMON::EBC_NUMBER - COMMON::WC_MAX_NUM - 1) + "','" + QString::number(0) + "')";
            query.exec(str);
        }

        ucTemp += COMMON::CC_FAULT_MAX;

        for (uint8_t i = ucTemp; i < ucTemp + COMMON::ATO_FAULT_MAX; i++)
        {
            QString strtemp = "CC(ATO)";
            QString str = "insert into LogCountTable(Type, ucIndex, LogCount) values('" + strtemp + "','" + QString::number(i - COMMON::EBC_NUMBER - COMMON::EBC_NUMBER - COMMON::WC_MAX_NUM - COMMON::CC_FAULT_MAX - 1) + "','" + QString::number(0) + "')";
            query.exec(str);
        }

        ///创建触发器，使得自动更新LogCountTable表中的LogCount值
        QSqlQuery TriggerLogCountquery(DBThreadConn);
        bool bSuccessTriggerLogCount = TriggerLogCountquery.exec("CREATE TRIGGER trigger_updateLogcount AFTER INSERT ON LogTable FOR EACH ROW WHEN new.id > 0 BEGIN update LogCountTable set LogCount = (select count(*) from LogTable where LogTable.ucIndex = new.ucIndex and LogTable.Type = new.Type) where LogCountTable.ucIndex = new.ucIndex and LogCountTable.Type = new.Type; END");
        if(bSuccessTriggerLogCount == false)
        {
            return ret = false;
        }
        ret = true;
    }
    else
    {
        TOD_LOG(TOD_LOG_WARNING, QString("[TOD] %1(TOD Datetime)[%2,LineNo:%3]:Create Database failed.") \
            .arg(QDateTime::currentDateTime().toString("yyyy-MM-dd hh:mm:ss.zzz")).arg(__FUNCTION__).arg(__LINE__));
        ret = false;
    }
    return ret;
}

/**
*    new fraction
*/
bool SqlInit::InitSql()
{
    QSqlQuery query(DBThreadConn);
    bool bSuccessLogCountTable = query.exec("create table LogCountTable (id INTEGER PRIMARY KEY, Type varchar(10), ucIndex varchar(10), LogCount varchar(10))");
    if(bSuccessLogCountTable)
    {
        uint8_t ucTemp = 1U;
		
        QString strtempEb = "EB";
        QString strunionEb;
        QString strEb = "INSERT INTO LogCountTable SELECT '" + QString::number(ucTemp) + "'AS 'id','"  + strtempEb +"'AS 'Type','" + QString::number(ucTemp) + "'AS 'ucIndex', '" + QString::number(0) + "'AS 'LogCount'";
        for (uint8_t i = ucTemp + 1; i < ucTemp + COMMON::EBC_NUMBER; i++)
        {
            strunionEb = "UNION ALL SELECT '" + QString::number(i) + "','" + strtempEb + "','" + QString::number(i) + "','" + QString::number(0) + "' ";
            strEb = strEb +strunionEb;
        }
        query.exec(strEb);
        ucTemp += COMMON::EBC_NUMBER;

        QString strtempFsb = "FSB";
        QString strunionfsb;
        QString strfsb = "INSERT INTO LogCountTable SELECT'"+ QString::number(ucTemp) + "'AS 'id','"  + strtempFsb +" 'AS 'Type','" + QString::number(ucTemp) + "'AS 'ucIndex', '" + QString::number(0) + "'AS 'LogCount'";
        for (uint8_t i = ucTemp + 1; i < ucTemp + COMMON::EBC_NUMBER; i++)
        {
            strunionfsb = "UNION ALL SELECT '"+ QString::number(i) + "','"  + strtempFsb + "','" + QString::number(i) + "','" + QString::number(0)+ "' ";
            strfsb = strfsb +strunionfsb;
        }
        query.exec(strfsb);
        ucTemp += COMMON::EBC_NUMBER;
        for (uint8_t i = ucTemp; i < ucTemp + COMMON::WC_MAX_NUM; i++)
        {
            QString strtemp = "Warning";
            QString str = "insert into LogCountTable(Type, ucIndex, LogCount) values('" + strtemp + "','" + QString::number(i - COMMON::EBC_NUMBER - COMMON::EBC_NUMBER) + "','" + QString::number(0) + "')";
            query.exec(str);
        }
        QString strtempWarning = "Warning";
        QString strunionwarning;
        QString strwarning = "INSERT INTO LogCountTable SELECT'" + QString::number(ucTemp) + "'AS 'id','" + strtempWarning +" 'AS 'Type','" + QString::number(ucTemp) + "'AS 'ucIndex' ,'" + QString::number(0) + "'AS 'LogCount'";
        for (uint8_t i = ucTemp + 1; i < ucTemp + COMMON::WC_MAX_NUM; i++)
        {
            strunionwarning = "UNION ALL SELECT '"+ QString::number(i) + "','"  + strtempWarning + "','" + QString::number(i) + "','" + QString::number(0)+ "' ";
            strwarning = strwarning +strunionwarning;
        }
        query.exec(strwarning);
        ucTemp += COMMON::WC_MAX_NUM;

        QString strtempCc = "CC";
        QString strunioncc;
        QString strcc = "INSERT INTO LogCountTable SELECT'" + QString::number(ucTemp) + "'AS 'id','"+ strtempCc +" 'AS 'Type','" + QString::number(ucTemp) + "'AS 'ucIndex', '" + QString::number(0) + "'AS 'LogCount'";
        for (uint8_t i = ucTemp + 1; i < ucTemp + COMMON::CC_FAULT_MAX; i++)
        {
            strunioncc = "UNION ALL SELECT '"+ QString::number(i) + "','" + strtempCc + "','" + QString::number(i) + "','" + QString::number(0) + "' ";
            strcc = strcc +strunioncc;
        }
        query.exec(strcc);
        ucTemp += COMMON::CC_FAULT_MAX;

        QString strtempCcAto = "CC(ATO)";
        QString strunionccato;
        QString strccato = "INSERT INTO LogCountTable SELECT'"+ QString::number(ucTemp) + "'AS 'id','" + strtempCcAto +" 'AS 'Type','" + QString::number(ucTemp) + "'AS 'ucIndex' ,'" + QString::number(0) + "'AS 'LogCount'";
        for (uint8_t i = ucTemp + 1; i < ucTemp + COMMON::CC_FAULT_MAX; i++)
        {
            strunionccato = "UNION ALL SELECT '" + QString::number(i) + "','" + strtempCcAto + "','" + QString::number(i) + "','" + QString::number(0) + "' ";
            strccato = strccato +strunionccato;
        }
        query.exec(strccato);
        ///创建触发器，使得自动更新LogCountTable表中的LogCount值
        QSqlQuery TriggerLogCountquery(DBThreadConn);
        bool bSuccessTriggerLogCount = TriggerLogCountquery.exec("CREATE TRIGGER trigger_updateLogcount AFTER INSERT ON LogTable FOR EACH ROW WHEN new.id > 0 BEGIN update LogCountTable set LogCount = (select count(*) from LogTable where LogTable.ucIndex = new.ucIndex and LogTable.Type = new.Type) where LogCountTable.ucIndex = new.ucIndex and LogCountTable.Type = new.Type; END");
        if(bSuccessTriggerLogCount == false)
        {
            return ret = false;
        }
        ret = true;
    }
    else
    {
        ret = false;
    }
    return ret;
}