一种通用数据采集的schema定义形式
{ "name": "凤凰金融", "notice": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ],
"comments": "网站通告"
}, "url": { "data": "attribute", "value": "http://www.fengjr.com/financing/list?type=cx"
"comments": "本平台数据的采集URL"
}, "project": { "data": "url", "url": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ], "template": "" }, "title": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ] }, "detail": { "title": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ] }, "amount": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ] } } }, "member": { "data": "sub_item", "sub_item": { "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ], "src-save": 0, "url": { "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ], "template": "" } }, "detail": { "title": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ] }, "amount": { "data": "attribute", "matcher": [ { "match": "xpath", "pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]" } ] } } }, "src-save": 1 }
补充：
{
"name": "凤凰金融",
"notice": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"url": {
"data": "attribute",
"value": "http://www.fengjr.com/financing/list?type=cx"
},
"project": {
"data": "url",
"url": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
},
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"detail": {
"name": "网贷列表",
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"amount": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
}
}
},
"member": {
"data": "sub_item",
"sub_item": {
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"src-save": 0,
"url": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
}
},
"detail": {
"name": "会员材料",
"title": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
},
"amount": {
"data": "attribute",
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
]
}
}
},
"src-save": 1,
"crawler": {
"handler":"httpClient|selenium",
"results":"html|json|text",
"next_page": {
"matcher": [
{
"match": "xpath",
"pattern": "//*[@id=\"page-financing\"]/div[1]/div[5]/div/div/div[3]"
}
],
"template": ""
},
"history": "re-crawl|skip|stop"
}
}