def peekNextToken
if @cur_pos == @page.length then return nil end
if ?< == @page[@cur_pos]
if '!--' == @page[(@cur_pos + 1), 3]
tag_end = @page.index('-->', (@cur_pos + 1))
if tag_end.nil?
raise HTMLTokenizerError, "No end found to started comment:\n#{@page[@cur_pos,80]}"
end
HTMLComment.new(@page[@cur_pos .. (tag_end + 2)])
else
tag_end = @page.index('>', (@cur_pos + 1))
if tag_end.nil?
raise HTMLTokenizerError, "No end found to started tag:\n#{@page[@cur_pos,80]}"
end
HTMLTag.new(@page[@cur_pos .. tag_end])
end
else
text_end = @page.index('<', @cur_pos)
text_end = text_end.nil? ? -1 : (text_end - 1)
HTMLText.new(@page[@cur_pos .. text_end])
end
end