batch批量获取数据和批量插入batchInsert工作总结
近期用到同步数据的工具,就是将一个网站的订单信息同步到一个后台订单系统中,由于订单数据量较大,
所以就用到了"批量" --- 批量获取 批量插入
1: 获取对象
$query = HmhOrder::find()
->select('id ,uid, order_no, remark, total_pay, pay_type, trade_no, order_from, create_at, update_at, pay_at, pay_status, is_del');
if ($starTime && $endTime) {
$query->andFilterWhere(['between', 'create_at', $starTime, $endTime]);
}
$query->with(['uid'=>function($query) {
return $query->select('contact_id, truename');
}])->asArray();2: batch的使用:
下面就是使用batch(), 每次取多少条数据,此时我设置的100条
foreach ($query->batch(100, Yii::$app->crmDb) as $rows) {
if (!$rows || !is_array($rows)) {
echo $this->stdout("无数据信息!\n", Console::BOLD);die; //这行是我用控制台输出的方式,让显示有点颜色
}
//遍历每一组数据，进行拆分各个字段，进行组合
foreach ($rows as $key => $value) {
//下面就是遍历你取出的每一条数据
......
}
}Yii::$app->crmDb 这是我的本地common/config/main-local.php文件下指向数据库以及数据库名字,密码,表前缀,等等
例如:
return [
'components' => [
'crmDb' => [
'class' => 'yii\db\Connection',
'dsn' => 'mysql:host=localhost;dbname=crm_test',
'username' => 'root',
'tablePrefix' => 'hm_',
'password' => '199285',
'charset' => 'utf8',
],
],
];
$result = Order::find()
->createCommand()
->batchInsert(Order::tableName(),$keys,$data)
->execute();
其中:
batchInsert() 里面有三个参数: 表名 要插入的表字段数组 要插入的表数据
$keys 是你要插入的表字段数组, 例如: ['name', 'age']
$data 是你要批量插入的数组集合(记得是一个二维数组哦!)
例如:[['Tom', 30],['Jane', 20],['Linda', 25]]@@ 下面是我的项目代码示例:
common/config/main-local.php配置文件:
[
'db' => [
'class' => 'yii\db\Connection',
'dsn' => 'mysql:host=localhost;dbname=oc_order',
'username' => 'root',
'password' => '199285',
'tablePrefix' => 'oc_',
'charset' => 'utf8',
],
'crmDb' => [
'class' => 'yii\db\Connection',
'dsn' => 'mysql:host=localhost;dbname=crm_test',
'username' => 'root',
'tablePrefix' => 'hm_',
'password' => '199285',
'charset' => 'utf8',
],
],
];Controller.php文件代码 : 主要就是同步工具代码
class SyncController extends Controller
{
/**
* 每次处理的记录数
*/
const PERPROC_NUM = 100;
/**
* 同步黑马入会订单信息|根据时间范围同步黑马入会订单信息
*/
public function actionHmhOrders($startTime = false, $endTime = false)
{
$data = [];
$mOrder = new Order();
//定义剩余数量,总数,已完成量
$surplus = 0;
$total = 0;
$num = 0;
//如果没有时间范围时，就是基本的导数据操作
if (!$startTime && !$endTime) {
//查询数据对象
$query = $this->ordersObj();
//计算总数
$total = $this->getNum();
} else {
//处理时间范围
$timeRange = $this->handleTimeRange($startTime, $endTime);
$startT = $timeRange['startTime'];
$endT = $timeRange['endTime'];
$query = $this->ordersObj($startT, $endT);
//计算总数
$total = $this->getNum($startT, $endT);
}
if (!$query) {
echo $this->ansiFormat("请求数据信息异常\n", Console::BOLD);die;
}
if ($total == 0) {
echo $this->ansiFormat("未查询导数据, 故停止同步!\n", Console::BOLD);die;
}
echo $this->ansiFormat("现在开始同步: -----> 共" . $total ."!\n", Console::BOLD);
//遍历查询订单信息
foreach ($query->batch(self::PERPROC_NUM, Yii::$app->crmDb) as $rows) {
if (!$rows || !is_array($rows)) {
echo $this->stdout("无数据信息!\n", Console::BOLD);die;
}
//遍历每一组数据，进行拆分各个字段，进行组合
foreach ($rows as $key => $value) {
$hmhData = $value;
$orderNo = date('ymd') . substr(time(), -5) . substr(microtime(), 2, 5) . sprintf('%02d', rand(0, 99));
$appOrderNo = isset($hmhData['order_no']) && $hmhData['order_no'] ? $hmhData['order_no'] : '';
$appOrderDesc = isset($hmhData['remark']) && $hmhData['remark'] ? $hmhData['remark'] : '';
$appId = 10000;
$productName = '黑马入会';
$turnover = isset($hmhData['total_pay']) && $hmhData['total_pay'] ? $hmhData['total_pay'] : 0.00;
$currency = 'cmy';
$channel = 0;
if (isset($hmhData['pay_type']) && $hmhData['pay_type']) {
if ($hmhData['pay_type'] == HmhOrder::CHANNEL_OFFLINE) {
$hmhData['pay_type'] = Order::CHANNEL_OFFLINE;
}
$channel = $hmhData['pay_type'];
}
$subChannel = 'wx';
$transactionNo = isset($hmhData['trade_no']) && $hmhData['trade_no'] ? $hmhData['trade_no'] : '';
$origin = isset($hmhData['order_from']) && $hmhData['order_from'] ? $hmhData['order_from'] : '';
$owner = '';
if (isset($hmhData['uid']['truename']) && $hmhData['uid']['truename']) {
$owner = $hmhData['uid']['truename'];
}
$createdAt = isset($hmhData['create_at']) && $hmhData['create_at'] ? $hmhData['create_at'] : 0;
$updatedAt = isset($hmhData['update_at']) && $hmhData['update_at'] ? $hmhData['update_at'] : 0;
$paiedAt = isset($hmhData['pay_at']) && $hmhData['pay_at'] ? $hmhData['pay_at'] : 0;
$payStatus = isset($hmhData['pay_status']) && $hmhData['pay_status'] ? $hmhData['pay_status'] : 0;
$status = isset($hmhData['is_del']) && $hmhData['is_del'] ? $hmhData['is_del'] : 1;
$data[] = [$orderNo, $appOrderNo, $appOrderDesc, $appId, $productName, $turnover,$currency, $channel, $subChannel, $transactionNo, $origin, $owner, $createdAt, $updatedAt,$paiedAt, $payStatus, $status];
//- 针对crm中的order_no，created_at对应到订单中心得app_order_no, app_id => 10000, created_at, 判断该条件下的订单是否存在，
$isOrder = $mOrder->isOrder($appOrderNo, $appId, $createdAt);
//- 存在 =》 跳过该条信息 不存在 =》 批量插入
if ($isOrder) {
echo $this->ansiFormat("订单号为:" . $appOrderNo ."的订单信息已存在, 故不同步!!!\n", Console::BOLD);
continue;
}
}
$keys = ['order_no', 'app_order_no', 'app_order_desc', 'app_id', 'product_name', 'turnover','currency', 'channel', 'sub_channel', 'transaction_no', 'origin', 'owner', 'created_at', 'updated_at', 'paied_at', 'pay_status', 'status'];
//批量插入
$result = $this->batchinsertOrders($keys, $data);
unset($data);
if (!$result) {
echo $this->ansiFormat("同步数据失败!\n", Console::BOLD);die;
}
//下面是我为了同步工具看似好看点,显示的已同步多少数据,剩余数据显示
if ($total <= self::PERPROC_NUM) {
$num = $num + $total;
$surplus = 0;
$total = 0;
} else {
$num = $num + self::PERPROC_NUM;
$surplus = $total - self::PERPROC_NUM;
$total = $surplus;
}
echo $this->ansiFormat("已同步数据" . $num . "条数据, 剩余" . $surplus . "条数据\n", Console::BOLD);
}
echo $this->ansiFormat("全部完成!\n", Console::BOLD);die;
}
/**
* 获取符合条件的订单信息总数
*
* @param bool|string $starTime 开始时间 默认 false
* @param bool|string $endTime 结束时间 默认 false
* @return int|string
*/
protected function getNum($starTime = false, $endTime = false)
{
$query = HmhOrder::find();
if ($starTime && $endTime) {
$query->andFilterWhere(['between', 'create_at', $starTime, $endTime]);
}
$result = $query->count();
return $result ? $result : 0;
}
/**
* 处理时间范围
*
* @param string $startTime 开始时间
* @param string $endTime 结束时间
* @return array
*/
protected function handleTimeRange($startTime, $endTime)
{
date_default_timezone_set('PRC');
if ($startTime && !$endTime) {
$endTime = strtotime($startTime .' 23:59:59');
$startTime = strtotime($startTime .' 00:00:00');
} else if ($startTime && $endTime) {
if ($startTime > $endTime) {
$tmp = $startTime;
$startTime = $endTime;
$endTime = $tmp;
}
$startTime = strtotime($startTime .' 00:00:00');
$endTime = strtotime($endTime .' 23:59:59');
}
$data = [
'startTime' => $startTime,
'endTime' => $endTime,
];
return $data;
}
/**
* 同步crm黑马入会订单表
*/
protected function ordersObj($starTime = false, $endTime = false)
{
$query = HmhOrder::find()
->select('id ,uid, order_no, remark, total_pay, pay_type, trade_no, order_from, create_at, update_at, pay_at, pay_status, is_del');
if ($starTime && $endTime) {
$query->andFilterWhere(['between', 'create_at', $starTime, $endTime]);
}
$query->with(['uid'=>function($query) {
return $query->select('contact_id, truename');
}])->asArray();
return $query ? $query : null;
}
/**
* 批量插入订单信息
*
* @param string $keys 要插入的字段属性集合
* @param array $data 要插入的字段值数组数据
* @return int|bool
*/
protected function batchinsertOrders($keys, $data)
{
$result = 0;
if ($keys && $data && is_array($data)) {
$result = Order::find()
->createCommand()
->batchInsert(Order::tableName(),$keys,$data)
->execute();
}
return $result ? $result : 0;
}
}我在项目中遇到个小问题就是:由于我们表设计有个业务订单号字段设计是"唯一", 所以那,我之前遇到了插入第一轮100条数据正常,但是下一轮数据时,就提示,order_no出现冲突,而且下一轮的order_no和前一轮的一样,这显然不正常,我们的业务订单号,是采用一种计算方式随机生成的,所以不会重复, 但是我反复执行了几次,还是出现重复错误, 最后发现,我做批量插入之前,不是需要插入数组集合吗?,就是下面这种的
$data[] = [$orderNo, $appOrderNo, $appOrderDesc, $appId, $productName......];
解决办法就是: unsert($data); 这样下一轮就是新的数组了.
项目工具截图:
