Pastable version of Can this scraper be simplified?

<div style="overflow:auto;border:solid 1px #ccc;background:#000;color:#F8F8F8">
 <div class="section">
 
 <pre style="float:left;margin:0 10px;border-right:0;color:#666;">1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73</pre>
 <pre class="sunburst">require 'rubygems'
require 'net/http'
require 'uri'

def getbody(letter, pagenumber)
 url = URI.parse('http://www.somesite.com/')
 res = Net::HTTP.start(url.host, url.port) { |http| 
 http.get(&quot;/browse.php?character=#{letter}&amp;page=#{pagenumber}&quot;)
 }
 gbody = res.body
 slice1 = gbody.index(&quot;columns&quot;)
 slice2 = gbody.rindex(&quot;&lt;center&gt;&quot;)
 gbody = gbody.slice(slice1, (slice2-slice1+150))
 return gbody
end

def getpages(htmlbody)
 pagebody = htmlbody.slice(htmlbody.index(&quot;&lt;center&gt;&quot;),50)
 slice1 = pagebody.index(&quot;&lt;b&gt;&quot;)
 slice2 = pagebody.index(&quot;&lt;/b&gt;&quot;)
 pages = pagebody.slice(slice1+3, (slice2-slice1-3))
 pages = Integer(pages)
 return pages
end

alphabet = %w(a b c d e f g h i j k l m n o p q r s t u v w x y z *)

alphacounter = 0;
allwords = Array.new
out = File.new('wordlist.html', 'w')
out.puts &quot;&lt;html&gt;&quot;

while alphacounter &lt; alphabet.length
 counter = 1
 cycles = 1
 while counter &lt;= cycles
 body = getbody(alphabet[alphacounter],(counter))
 if counter == 1
 if alphabet[alphacounter] == &quot;s&quot;
 cycles = 1010
 else
 cycles = getpages(body)
 end
 puts &quot;cycles: #{cycles}&quot;
 end
 counter += 1
 wordbody = body.slice(0,body.index(&quot;&lt;/table&gt;&quot;))
 peices = Array.new
 innercounter = 0;
 while (wordbody.include? &quot;&lt;a href&quot;)
 slice1 = wordbody.index(&quot;&lt;a href&quot;)
 slice2 = wordbody.index(&quot;&lt;/a&gt;&quot;)
 peices[innercounter] = wordbody.slice(slice1,(slice2-slice1+4))
 innercounter += 1
 wordbody = wordbody.slice((slice2+4),wordbody.length)
 end
 peices.compact!
 i = 0
 while i &lt; peices.length
 peices[i].insert((peices[i].index(&quot;=&quot;)+2),&quot;http://www.somesite.com&quot;)
 i += 1
 end
 peices.each do |item|
 out.puts &quot;#{item} &lt;br&gt;&quot;
 end
 puts &quot;Letter: #{alphabet[alphacounter]} page: #{counter} total pages: #{cycles}&quot;
 end
 alphacounter += 1 
end

out.puts &quot;&lt;/html&gt;&quot;

</pre>
 </div>
 </div>

Refactor :my => 'code'

Recent

Popular

Pastable version of Can this scraper be simplified?