description = [[
Returns a list of URIs found in the web server. 

This script depends on the library "httpspider" to crawl the webserver. If you would like to include binary files in the results use <code>httpspider.showBinaries</code>. 
To treat URIs with parameters as if they were the same, use <code>httpspider.ignoreParams</code>
]]

---
-- @usage
-- nmap -sV --script http-sitemap <target>
-- nmap -p80 --script http-sitemap --script-args http.useragent=Mozilla,httpspider.ignoreParams <target>
-- @output
--PORT   STATE SERVICE REASON
--80/tcp open  http
--| http-sitemap: URIs found:
--|_http://scanme.nmap.org/
--
-- Other useful args:
-- http.useragent - User Agent for the HTTP requests
-- httpspider.showBinaries - Includes binary files in results
-- httpspider.ignoreParams - Ignores parameters and treat URIs as if they were the same.
---

author = "Paulino Calderon"
license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
categories = {"discovery"}

require "http"
require "shortport"
require "httpspider"

portrule = shortport.http

action = function(host, port)
  local results = {"URIs found:"}

  httpspider.crawl(host, port)

  local uris = httpspider.get_sitemap()
  for _, uri in pairs(uris) do
    results[#results+1] = uri
  end

  return #results > 1 and stdnse.strjoin("\n", results) or nil
end