thread_num=100
cat $1| xargs -n 1 -I {} -P ${thread_num} sh url_crawler.sh {}
url_crawler.sh
time=`date +"%Y%m%d%H%M"`
line=$1
http_code=`curl -sI -w "%{http_code}" -o /dev/null --connect-timeout 1 -m 1 $line`
echo -e "$http_code\t$line"
#echo -e "$http_code\t$line" > ./data/spider_log.$time
