PHP获取网页内容的几种方法及内容分析

2017年7月13日 By yuzhou | 分类: Lnmp | views

1、获取内容的方法：

方法1: 用file_get_contents 以get方式获取内容

<?php

$url=‘http://www.domain.com/?para=123’;

$html = file_get_contents($url);

echo $html;

?>

方法2：用file_get_contents函数,以post方式获取url

<?php

$url = ‘http://www.domain.com/test.php?id=123’;

$data = array (‘foo’ => ‘bar’);

$data = http_build_query($data);

$opts = array (

‘http’ => array (

   ‘method’ => ‘POST’,

   ‘header’=> “Content-type: application/x-www-form-urlencoded\r\n” .

                     “Content-Length: “ . strlen($data) . “\r\n”,

   ‘content’ => $data

)

);

$ctx = stream_context_create($opts);

$html = @file_get_contents($url,”,$ctx);

如果需要再传递cookie数据,则把

‘header’=> “Content-type: application/x-www-form-urlencoded\r\n” .

                  “Content-Length: “ . strlen($data) . “\r\n”,

修改为

‘header’=> “Content-type: application/x-www-form-urlencoded\r\n” .

                 “Content-Length: “ . strlen($data) . “\r\n”.

                 “cookie:cookie1=c1;cookie2=c2\r\n” ;

即可

方法3: 用fopen打开url, 以get方式获取内容

<?php

$fp = fopen($url, ‘r’);

$header = stream_get_meta_data($fp);//获取报头信息

while(!feof($fp)) {

$result .= fgets($fp, 1024);

}

echo “url header: {$header} <br>”:

echo “url body: $result“;

fclose($fp);

?>

方法4: 用fopen打开url, 以post方式获取内容

<?php

$data = array (‘foo2’ => ‘bar2’,‘foo3’=>‘bar3’);

$data = http_build_query($data);

$opts = array (

‘http’ => array (

‘method’ => ‘POST’,

‘header’=> “Content-type: application/x-www-form-urlencoded\r\nCookie:cook1=c3;cook2=c4\r\n” .

“Content-Length: “ . strlen($data) . “\r\n”,

‘content’ => $data

)

);

$context = stream_context_create($opts);

$html = fopen(‘http://www.test.com/zzzz.php?id=i3&id2=i4’,‘rb’ ,false, $context);

$w=fread($html,1024);

echo $w;

?>

方法5：用fsockopen函数打开url，以get方式获取完整的数据，包括header和body

<?php

function get_url ($url,$cookie=false)

{

$url = parse_url($url);

$query = $url[path].“?”.$url[query];

echo “Query:”.$query;

$fp = fsockopen( $url[host], $url[port]?$url[port]:80 , $errno, $errstr, 30);

if (!$fp) {

return false;

} else {

$request = “GET $query HTTP/1.1\r\n”;

$request .= “Host: $url[host]\r\n”;

$request .= “Connection: Close\r\n”;

if($cookie) $request.=“Cookie:   $cookie\n”;

$request.=“\r\n”;

fwrite($fp,$request);

while(!@feof($fp)) {

$result .= @fgets($fp, 1024);

}

fclose($fp);

return $result;

}

}

//获取url的html部分，去掉header

function GetUrlHTML($url,$cookie=false)

{

$rowdata = get_url($url,$cookie);

if($rowdata)

{

$body= stristr($rowdata,“\r\n\r\n”);

$body=substr($body,4,strlen($body));

return $body;

}

   return false;

}

?>

方法6：用fsockopen函数打开url，以POST方式获取完整的数据，包括header和body

<?php

function HTTP_Post($URL,$data,$cookie, $referrer=“”)

{

   // parsing the given URL

$URL_Info=parse_url($URL);

   // Building referrer

if($referrer==“”) // if not given use this script as referrer

$referrer=“111”;

   // making string from $data

foreach($data as $key=>$value)

$values[]=“$key=”.urlencode($value);

$data_string=implode(“&”,$values);

   // Find out which port is needed – if not given use standard (=80)

if(!isset($URL_Info[“port”]))

$URL_Info[“port”]=80;

   // building POST-request:

$request.=“POST “.$URL_Info[“path”].” HTTP/1.1\n”;

$request.=“Host: “.$URL_Info[“host”].“\n”;

$request.=“Referer: $referer\n”;

$request.=“Content-type: application/x-www-form-urlencoded\n”;

$request.=“Content-length: “.strlen($data_string).“\n”;

$request.=“Connection: close\n”;

   $request.=“Cookie:   $cookie\n”;

   $request.=“\n”;

$request.=$data_string.“\n”;

   $fp = fsockopen($URL_Info[“host”],$URL_Info[“port”]);

fputs($fp, $request);

while(!feof($fp)) {

$result .= fgets($fp, 1024);

}

fclose($fp);

   return $result;

}

?>

方法7:使用curl库，使用curl库之前，可能需要查看一下php.ini是否已经打开了curl扩展

<?php

$ch = curl_init();

$timeout = 5;

curl_setopt ($ch, CURLOPT_URL, ‘http://www.domain.com/’);

curl_setopt ($ch, CURLOPT_RETURNTRANSFER, 1);

curl_setopt ($ch, CURLOPT_CONNECTTIMEOUT, $timeout);

$file_contents = curl_exec($ch);

curl_close($ch);

echo $file_contents;

?>

2、将获取到的内容进行分析

<?php

//获取所有内容url保存到文件

function get_index ( $save_file , $prefix = “index_” ){

   $count = 68 ;

   $i = 1 ;

if ( file_exists ( $save_file )) @ unlink ( $save_file );

   $fp = fopen ( $save_file , “a+” ) or die( “Open “ . $save_file . ” failed” );

while( $i < $count ){

     $url = $prefix . $i . “.htm” ;

    echo “Get “ . $url . “…” ;

     $url_str = get_content_url ( get_url ( $url ));

    echo ” OK/n” ;

     fwrite ( $fp , $url_str );

    ++ $i ;

}

   fclose ( $fp );

}

//获取目标多媒体对象

function get_object ( $url_file , $save_file , $split = “|–:**:–|” ){

if (! file_exists ( $url_file )) die( $url_file . ” not exist” );

   $file_arr = file ( $url_file );

if (! is_array ( $file_arr ) || empty( $file_arr )) die( $url_file . ” not content” );

   $url_arr = array_unique ( $file_arr );

if ( file_exists ( $save_file )) @ unlink ( $save_file );

   $fp = fopen ( $save_file , “a+” ) or die( “Open save file “ . $save_file . ” failed” );

foreach( $url_arr as $url ){

    if (empty( $url )) continue;

    echo “Get “ . $url . “…” ;

     $html_str = get_url ( $url );

    echo $html_str ;

    echo $url ;

    exit;

     $obj_str = get_content_object ( $html_str );

    echo ” OK/n” ;

     fwrite ( $fp , $obj_str );

}

   fclose ( $fp );

}

//遍历目录获取文件内容

function get_dir ( $save_file , $dir ){

   $dp = opendir ( $dir );

if ( file_exists ( $save_file )) @ unlink ( $save_file );

   $fp = fopen ( $save_file , “a+” ) or die( “Open save file “ . $save_file . ” failed” );

while(( $file = readdir ( $dp )) != false ){

    if ( $file != “.” && $file != “..” ){

      echo “Read file “ . $file . “…” ;

       $file_content = file_get_contents ( $dir . $file );

       $obj_str = get_content_object ( $file_content );

      echo ” OK/n” ;

       fwrite ( $fp , $obj_str );

    }

}

   fclose ( $fp );

}

//获取指定url内容

function get_url ( $url ){

   $reg = ‘/^http:////[^//].+$/’ ;

if (! preg_match ( $reg , $url )) die( $url . ” invalid” );

   $fp = fopen ( $url , “r” ) or die( “Open url: “ . $url . ” failed.” );

while( $fc = fread ( $fp , 8192 )){

     $content .= $fc ;

}

   fclose ( $fp );

if (empty( $content )){

    die( “Get url: “ . $url . ” content failed.” );

}

return $content ;

}

//使用socket获取指定网页

function get_content_by_socket ( $url , $host ){

   $fp = fsockopen ( $host , 80 ) or die( “Open “ . $url . ” failed” );

   $header = “GET /” . $url . ” HTTP/1.1/r/n” ;

   $header .= “Accept: */*/r/n” ;

   $header .= “Accept-Language: zh-cn/r/n” ;

   $header .= “Accept-Encoding: gzip, deflate/r/n” ;

   $header .= “User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; Maxthon; InfoPath.1; .NET CLR 2.0.50727)/r/n” ;

   $header .= “Host: “ . $host . “/r/n” ;

   $header .= “Connection: Keep-Alive/r/n” ;

   //$header .= “Cookie: cnzz02=2; rtime=1; ltime=1148456424859; cnzz_eid=56601755-/r/n/r/n”;

   $header .= “Connection: Close/r/n/r/n” ;

   fwrite ( $fp , $header );

   while (! feof ( $fp )) {

     $contents .= fgets ( $fp , 8192 );

   }

   fclose ( $fp );

   return $contents ;

}

//获取指定内容里的url

function get_content_url ( $host_url , $file_contents ){

   //$reg = ‘/^(#|<a href=”http://lib.csdn.net/base/18″ class=’replace_word’ title=”JavaScript知识库” target=’_blank’ style=’color:#df3434; font-weight:bold;’>JavaScript</a>.*?|ftp:////.+|http:////.+|.*?href.*?|play.*?|index.*?|.*?asp)+$/i’;

   //$reg = ‘/^(down.*?/.html|/d+_/d+/.htm.*?)$/i’;

   $rex = “/([hH][rR][eE][Ff])/s*=/s*[‘/”]*([^>‘/”/s]+)[/”‘>]*/s*/i” ;

   $reg = ‘/^(down.*?/.html)$/i’ ;

   preg_match_all ( $rex , $file_contents , $r );

   $result = ““ ; //array();

   foreach( $r as $c ){

    if ( is_array ( $c )){

      foreach( $c as $d ){

        if ( preg_match ( $reg , $d )){ $result .= $host_url . $d . “/n“ ; }

      }

    }

}

return $result ;

}

//获取指定内容中的多媒体文件

function get_content_object ( $str , $split = “|–:**:–|” ){

   $regx = “/href/s*=/s*[‘/”]*([^>’/“/s]+)[/”‘>]*/s*(.*?<//b>)/i” ;

   preg_match_all ( $regx , $str , $result );

if ( count ( $result ) == 3 ){

     $result [ 2 ] = str_replace ( “多媒体： ” , “” , $result [ 2 ]);

     $result [ 2 ] = str_replace ( ” ” , “” , $result [ 2 ]);

     $result = $result [ 1 ][ 0 ] . $split . $result [ 2 ][ 0 ] . “/n” ;

}

return $result ;

}

?>

参考链接：

1）https://www.oschina.net/code/snippet_861770_19638

2）http://www.jb51.net/article/83041.htm

发表评论 取消回复

发表评论取消回复