Lua

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Lua

local chilkat = require("chilkat")

local success = false

local spider = chilkat.Spider.new()

local seenDomains = chilkat.StringArray.new()
local seedUrls = chilkat.StringArray.new()

seenDomains.unique = true
seedUrls.unique = true

-- You will need to change the start URL to something else...
seedUrls:append("http://something.whateverYouWant.com/")

-- Set outbound URL exclude patterns
-- URLs matching any of these patterns will not be added to the
-- collection of outbound links.
spider:add_avoid_outbound_link_pattern("*?id=*")
spider:add_avoid_outbound_link_pattern("*.mypages.*")
spider:add_avoid_outbound_link_pattern("*.personal.*")
spider:add_avoid_outbound_link_pattern("*.comcast.*")
spider:add_avoid_outbound_link_pattern("*.aol.*")
spider:add_avoid_outbound_link_pattern("*~*")

-- Use a cache so we don't have to re-fetch URLs previously fetched.
spider.cache_dir = "c:/spiderCache/"
spider.fetch_from_cache = true
spider.update_cache = true

while seedUrls.count > 0 do
    local url = seedUrls:pop()
    spider:initialize(url)

    -- Spider 5 URLs of this domain.
    -- but first, save the base domain in seenDomains
    local domain = spider:get_url_domain(url)
    seenDomains:append(spider:get_base_domain(domain))

    local numCrawled = 0
    success = true
    while (success) and (numCrawled < 5) do
        success = spider:crawl_next()
        if success then
            -- Display the URL we just crawled.
            print(spider.last_url)

            -- If the last URL was retrieved from cache,
            -- we won't wait.  Otherwise we'll wait 1 second
            -- before fetching the next URL.
            if not spider.last_from_cache then
                spider:sleep_ms(1000)
            end

            numCrawled = numCrawled + 1
        end

        -- If CrawlNext fails (no more URLs to crawl in this domain), success is false and the loop exits.
    end

    -- Add the outbound links to seedUrls, except
    -- for the domains we've already seen.
    for i = 0, spider.num_outbound_links - 1 do
        url = spider:get_outbound_link(i)
        domain = spider:get_url_domain(url)
        local baseDomain = spider:get_base_domain(domain)
        if not seenDomains:contains(baseDomain) then
            -- Don't let our list of seedUrls grow too large.
            if seedUrls.count < 1000 then
                seedUrls:append(url)
            end
        end
    end
end