将之前爬虫抽取器用aardio实现。
import myplu.jpath;
import myplu.xpath;
namespace myplu
// 选择器基类
// 继承该类实现handle方法，handle每次处理一条数据
class selector{
ctor( ... ){
this.expression = "";
this.expType = "MIX";
if(type(...) == "table"){
..table.assign(this,...);
}
};
type="crawler.selector";
["select"] = function(data){
var res = owner.selectList(data);
if(#res and table.len(res)>0){
return res[1];
}
return res;
};
selectList = function(data){
// 空数据不用处理，直接退出
if(data == null){
return null;
}
var res = {};
// 将单个数据转为数组来处理
if(!(type(data) == "table" and ..table.len(data)>0)){
data = {data};
}
// 遍历数据，依次处理
for(k,v in data){
// 执行handle方法,抽取数据。
var dt = owner.handle(v);
if(dt){
// 处理数据结果，数组合并，非数组放入数组。
if(type(dt) == "table" and ..table.len(dt)>0){
..table.assign(res,dt);
}else{
..table.push(res,dt);
}
}
}
// 如果结果就一条，直接返回数据值。
if(#res == 1){
return res[1];
}
return res;
};
parse = function(...){
..table.assign(this,...);
};
category = {"MIX","XPATH","JPATH","REGEX","SPLIT","REPLACE"};
space = {"URL","BODY","BLOCK","PAGE"};
}
// 模式提取选择器
class regexSelector{
ctor( expression, subStr){
this = ..myplu.selector(expression=expression; subStr=subStr; expType = "REGEX");
};
type="crawler.regexSelector";
handle = function(data){
var data = ..string.matches(data,this.expression);
if(type(subStr) == "number" and subStr > 0 and subStr < #data){
return data[subStr];
}
return data;
};
}
// 模式拆分选择器
class splitSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "SPLIT");
};
type="crawler.splitSelector";
handle = function(data){
return ..string.split(data,this.expression);
};
}
// 模式替换选择器
class replaceSelector{
ctor( expression, subStr){
this = ..myplu.selector(expression=expression; subStr=subStr; expType = "REPLACE");
};
type="crawler.replaceSelector";
handle = function(data){
return ..string.replace(data,this.expression,this.subStr);
};
}
// xpath选择器
class xpathSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "XPATH");
};
type="crawler.xpathSelector";
handle = function(data){
data = ..myplu.xpath(data).selectList(this.expression);
if(#data==1){
return data[1];
}
return data;
};
}
// jpath选择器
class jpathSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "JPATH");
};
type="crawler.jpathSelector";
handle = function(data){
data = ..myplu.jpath(data).selectList(this.expression);
if(#data==1){
return data[1];
}
return data;
};
}
// 混合选择器
class mixSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "MIX");
};
type="crawler.mixSelector";
handle = function(data){
if(!this["selectors"]){
var exp = ..string.replace(this.expression,"(\w+)(%())", "..myplu.\1Selector\2");
exp = ..string.matches(exp, "\.\.[^\.][\w\W]+?%()");
this.selectors = {};
for(k,v in exp){
var sel = eval(v[1]);
..table.push(this.selectors,sel);
}
}
for(k,v in this.selectors){
data = v.handle(data);
}
return data;
};
}
// 链式抽取器
class selectors{
ctor(){
this.selectors = {};
};
type="crawler.selectors";
jpath = function(expression){
..table.push(this.selectors, ..myplu.jpathSelector(expression,araeSpace,mutil));
return this;
};
xpath = function(expression){
..table.push(this.selectors, ..myplu.xpathSelector(expression,araeSpace,mutil));
return this;
};
regex = function(expression, subStr){
..table.push(this.selectors, ..myplu.regexSelector(expression,subStr,araeSpace,mutil));
return this;
};
replace = function(expression, subStr){
..table.push(this.selectors, ..myplu.replaceSelector(expression,subStr,araeSpace,mutil));
return this;
};
split = function(expression){
..table.push(this.selectors, ..myplu.splitSelector(expression,araeSpace,mutil));
return this;
};
mix = function(expression){
..table.push(this.selectors, ..myplu.mixSelector(expression,araeSpace,mutil));
return this;
};
more = function(...){
var data = ...;
if(type(data)!="table"){
data = {...};
}elseif(data["type"]) {
data = {...};
}
..table.push(this.selectors,data);
return this;
};
["select"] = function(data){
data = this.selectList(data);
if(#data){
return data[1];
}
};
selectList = function(data){
for(k,v in this.selectors){
if(v["type"] and ..string.find(v["type"],"crawler\.\w+Selector")){
data = v.selectList(data);
}else {
var res = {};
//..console.dump(data);
if(!(type(data) == "table" and ..table.len(data)>0)){
data = {data};
}
//..console.dump(data);
for(k1, v1 in data){
var re = {};
for(k2, v2 in v){
//..console.dump(v2);
if(v2["type"] and v2["type"]=="crawler.selectors"){
re[k2] = v2.selectList(v1);
}else{
// 附加的直接传值
re[k2] = v2;
}
}
..table.push(res,re);
}
data = res;
}
if(data == null){
return null;
}
}
return data;
};
}
namespace crawler.selector
/*****intellisense(myplu)
crawler.selector = 导入数据抽取器
selector(__) = 基础选择器。需要自己实现handle方法，执行数据在处理函数。\n!cwSelector.
regexSelector(.(表达式, 抽取结果下标) = 模式匹配选择器。\n!cwSelector.
replaceSelector(.(表达式, 替换内容) = 模式替换选择器。\n!cwSelector.
splitSelector(.(表达式) = 模式拆分选择器。\n!cwSelector.
xpathSelector(.(表达式) = xpath选择器。\n!cwSelector.
jpathSelector(.(表达式) = jsonPath选择器。\n!cwSelector.
mixSelector(.(表达式) = 混合选择器。\n!cwSelector.
selectors() = 串联选择器。\n!cwSelectors.
end intellisense*****/
/*****intellisense(!cwSelector)
select(__) = 执行选择器，抽取单条数据。
selectList(__) = 执行选择器，抽取数据列表。
handle(__) = 选择器处理函数。
parse(__) = 将table转为对象。
category.? = 选择器类型
space.? = 抽取数据区域
subStr = 替换选择器，替换内容。
expression = 规则表达式。
expType = 选择器类型category.?。\n"MIX":混合选择器,\n"XPATH":xpath选择器,\n"JPATH":jsonPath选择器,\n"REGEX":模式抽取选择器,\n"SPLIT":模式拆分选择器,\n"REPLACE":模式替换选择器。
setExpression(__) = 设置规则表达式。
setExpType(__) = 选择器类型category.?。\n"MIX":混合选择器,\n"XPATH":xpath选择器,\n"JPATH":jsonPath选择器,\n"REGEX":模式抽取选择器,\n"SPLIT":模式拆分选择器,\n"REPLACE":模式替换选择器。
setSubStr(__) = 设置替换内容。
getExpression() = 获取规则表达式。
getExpType() = 选择器类型。
getSubStr() = 获取替换内容。
end intellisense*****/
/*****intellisense(!cwSelectors)
regex(.(表达式,抽取结果下标) = 模式匹配选择器。\n!cwSelectors.
split(.(表达式) = 模式拆分选择器。\n!cwSelectors.
replace(.(表达式,替换内容) = 模式替换选择器。\n!cwSelectors.
xpath(.(表达式) = xpath选择器。\n!cwSelectors.
jpath(.(表达式) = jsonPath选择器。\n!cwSelectors.
mix(.(表达式) = 混合选择器。\n!cwSelectors.
more(.(__) = 列表抽取。@1 可以是myplu.selectors数组，或myplu.selectors为value的表
select(__) = 执行选择器，抽取单条数据。
selectList(__) = 执行选择器，抽取数据列表。
end intellisense*****/
