多线程抓取网站内容代码,PHP代码库,德仔网

代码语言

代码分类

【PHP】多线程抓取网站内容

作者:李敬塔 / 发布于2012/12/5/ 888

/**
** @author: 李敬塔(Tiger)
** @desc: 多线程抓取网站内容
** @createDate:2011-08-11
**/
class MultiHttpRequest{
        public $urls = array();
        public $curlopt_header = 1;
        public $method = "GET";
        function __construct($urls = false){
                $this->urls = $urls;
        }
        function set_urls($urls){
                $this->urls = $urls;
                return $this;
        }
        function is_return_header($b){
                $this->curlopt_header = $b;
                return $this;
        }
        function set_method($m){
                $this->medthod = strtoupper($m);
                return $this;
        }
        function start(){
             $req_text="";
                if(!is_array($this->urls) or count($this->urls) == 0){
                        return false;
                }
                $curl = $text = array();
                $handle = curl_multi_init();
                foreach($this->urls as $k=>$v){
                        $curl[$k] = $this->add_handle($handle, $v);
                }
                $this->exec_handle($handle);
                foreach($this->urls as $k=>$v){
                        $text[$k] =  curl_multi_getcontent ($curl[$k]);
                        $req_text.=$text[$k];
                       curl_multi_remove_handle($handle, $curl[$k]);
                }
                curl_multi_close($handle);
                return $req_text;
        }
        private function add_handle(&$handle, $url)
        {
                $curl = curl_init();
                curl_setopt($curl, CURLOPT_URL, $url);
                curl_setopt($curl, CURLOPT_HEADER, $this->curlopt_header);
                curl_setopt($curl, CURLOPT_RETURNTRANSFER, 1);
                curl_multi_add_handle($handle, $curl);
                return $curl;
        }
        private function exec_handle(&$handle)
        {
                $flag = null;
                do {
                        curl_multi_exec($handle, $flag);
                } while ($flag > 0);
        }
}
$urls = array("http://www.php.net/",http://www.baidu.com);
$mp = new MultiHttpRequest($urls);
$req_text=$mp->start();
preg_match('/<div[^>]*id="headmenu"[^>]*>(.*?) <\/div>/si',$req_text,$match); //得到某区域的内容  
print_r($match[1]);echo "<br/>";
//echo $req_text;

试试其它关键字

　抓取网站内容　

同语言下

可能有用的

李敬塔贡献的其它代码(13)