Pastable version of Robots Reader

<div style="overflow:auto;border:solid 1px #ccc;background:#000;color:#F8F8F8">
 <div class="section">
 
 <pre style="float:left;margin:0 10px;border-right:0;color:#666;">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45</pre>
 <pre class="sunburst">&lt;?
function robots_allowed($url){
	$current_url=$url;
	$xmp=explode(&quot;/&quot;, $current_url.&quot;/&quot;);
	$robotsdomain=trim(&quot;http://&quot;.$xmp[2]);
	$stipped_robotsdomain=str_replace(&quot;/&quot;,&quot;&quot;,$robotsdomain);
	$stripped_current_url=str_replace(&quot;/&quot;, &quot;&quot; ,$url); 
	$my_user_agent=&quot;User-agent: intermap&quot;; //my useragent 
	$robots=Read_Content($robotsdomain.'/robots.txt'); 
	$robots=explode(&quot;\n&quot;,$robots); 
	for ($i=0;$i&lt;sizeof($robots);$i++){ 
		if (trim($robots[$i])==$my_user_agent){ // rules for agent: * 
			for ($checkrules=1;$checkrules&lt;10;$checkrules++){ 
				if (trim($robots[$i+$checkrules])!=&quot;&quot;){ 
					$pos = strpos( $current_line[$count],&quot;User-agent&quot;); 
					if (is_integer($pos)) break; 
					$pos = strpos( $current_line[$count],&quot;#&quot;); 
					if (is_integer($pos)) $current_line[$count]=substr($current_line[$count],0,$pos); 
					$disallow_line=str_replace(&quot;Disallow: &quot;, &quot;&quot; ,$robots[$i+$checkrules]); 
					//$disallow_line=str_replace(&quot;http://&quot;, &quot;&quot; ,$disallow_line); 
					$disallow_line=str_replace(&quot;/&quot;, &quot;&quot; ,$disallow_line);
					$newdata[$num]=$stipped_robotsdomain.$disallow_line;
					$num++;
					$count++;
				}
			}
		}
	}
	$forbidden=1; 
	for ($last=0;$last&lt;20;$last++){ 
		if (trim($newdata[$last])!=&quot;&quot;){ 
			if (preg_match(&quot;/&quot;.trim($newdata[$last]).&quot;/i&quot;,$stripped_current_url)) {$forbidden=0;} 
		} 
	} 
	return $forbidden; 
} 
function Read_Content($url){// Open een url return content 
	$handle=@fopen($url,&quot;r&quot;); 
	if($handle){ 
		$contents = fread ($handle, 10000); 
		fclose($handle); 
	} 
	return $contents; 
}
?&gt;
</pre>
 </div>
 </div>

Refactor :my => 'code'

Recent

Popular

Pastable version of Robots Reader