description = [[ 
Crawls through archive.org and extracts previous versions for the target website.
]]

--- 
-- @usage nmap -p80 --script http-archive.nse <target>  
--
-- @args http-archive.maxyears The maximum number of archived years to search
--       through. The years start from the past. For example, if a webpage is being
--       archived since 2000 and we set the value of maxyears to 10, the script will
--       return the archived versions (one per year) until 2010. Default: 15
--
-- @output
-- PORT   STATE SERVICE REASON
-- 80/tcp open  http    syn-ack
-- | http-archive
-- |   1998: web.archive.org/web/19981212025927/http://www.example.com
-- |   1999: web.archive.org/web/19990422132409/http://www.example.com
-- |   2000: web.archive.org/web/20001206131200/http://www.example.com
-- |   2001: web.archive.org/web/20010710052043/http://www.example.com
-- |   2002: web.archive.org/web/20021124055233/http://www.example.com
-- |_  2003: web.archive.org/web/20031214175554/http://www.example.com
---

author = {'George Chatzisofroniou'}
license = "Same as Nmap--See http://nmap.org/book/man-legal.html"
categories = {"intrusive", "external", "discovery"}

local nmap = require "nmap"
local http = require "http"
local httpspider = require "httpspider"
local shortport = require "shortport"
local stdnse = require "stdnse"
local table = require "table"
local string = require "string"

portrule = shortport.port_or_service( {80, 443}, {"http", "https"}, "tcp", "open")

local ARCHIVE_SITE = "web.archive.org"
local ARCHIVE_YEAR = "%s(%d%d%d%d)</a>"

local removewww = function(url) return string.gsub(url, "^www%.", "") end

action = function(host, port)

    local maxyears = stdnse.get_script_args("http-archive.maxyears") or 15
    local singleyears = stdnse.get_script_args("http-archive.singleyears") or nil

    local urls = {}
    local target = "/web/*/" .. host.targetname

    -- Only one instantiation of the script should ping archive.org at once.
    local mutex = nmap.mutex("http-archive")
    mutex "lock"

    -- Get the first archived year.
    local response = http.get(ARCHIVE_SITE, 80, target)
    local year = string.match(response.body, ARCHIVE_YEAR)

    -- If you can't find it, the target website is not archived.
    if not year then
        return "archive.org doesn't contain any archived version of this website."
    end

    local yearscount = 0
    local curyear = tonumber(os.date("%Y", nmap.clock()))
    local oldurl = ""    
    local checkedurls = {}
    local targetname = removewww(host.targetname)
    local index, k
    while true do

        if singleyears then
            k, url = next(singleyears, index)

            if (k == nil) then
                break
            end

            -- If the table contains years.
            if type(tonumber(url)) == "number" then
                target = "/web/" .. url .. "1000000000*/" .. host.targetname 
                local response = http.get(ARCHIVE_SITE, 80, target)

                -- Get the last archived version for this year.
                url = string.match(response.body, "<a href=\"(/web/%d*/https?://www%." .. targetname .. "/.-)\"") or 
                  string.match(response.body, "<a href=\"(/web/%d*/https?://" .. targetname .. "/.-)\"") or ""
            end

            if (index) then
                index = index + 1
            else 
                index = 1
            end
        else

            -- If we passed current year or the max limit.
            if year and tonumber(year) > curyear and yearscount > maxyears then
                break
            end

            stdnse.print_debug(2, "Checking archived version in " .. year)

            target = "/web/" .. year .. "1000000000*/" .. host.targetname
            response = http.get(ARCHIVE_SITE, 80, target)

            if not response.body then
                break
            end

            -- Get the last archived version for this year.
            url = string.match(response.body, ".*<a href=\"(/web/%d*/https?://www%." .. targetname .. "/.-)\"") or 
                  string.match(response.body, ".*<a href=\"(/web/%d*/https?://" .. targetname .. "/.-)\"") or ""


            stdnse.print_debug(2, "URL: " .. url)

            table.insert(urls, year .. ": web.archive.org" .. url)

            year = year + 1
            yearscount = yearscount + 1
        end
    end
    mutex "done"

    return urls

end