PureBasic

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. PureBasic

IncludeFile "CkStringArray.pb"
IncludeFile "CkSpider.pb"

Procedure ChilkatExample()

    success.i = 0

    spider.i = CkSpider::ckCreate()
    If spider.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    seenDomains.i = CkStringArray::ckCreate()
    If seenDomains.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    seedUrls.i = CkStringArray::ckCreate()
    If seedUrls.i = 0
        Debug "Failed to create object."
        ProcedureReturn
    EndIf

    CkStringArray::setCkUnique(seenDomains, 1)
    CkStringArray::setCkUnique(seedUrls, 1)

    ; You will need to change the start URL to something else...
    CkStringArray::ckAppend(seedUrls,"http://something.whateverYouWant.com/")

    ; Set outbound URL exclude patterns
    ; URLs matching any of these patterns will not be added to the 
    ; collection of outbound links.
    CkSpider::ckAddAvoidOutboundLinkPattern(spider,"*?id=*")
    CkSpider::ckAddAvoidOutboundLinkPattern(spider,"*.mypages.*")
    CkSpider::ckAddAvoidOutboundLinkPattern(spider,"*.personal.*")
    CkSpider::ckAddAvoidOutboundLinkPattern(spider,"*.comcast.*")
    CkSpider::ckAddAvoidOutboundLinkPattern(spider,"*.aol.*")
    CkSpider::ckAddAvoidOutboundLinkPattern(spider,"*~*")

    ; Use a cache so we don't have to re-fetch URLs previously fetched.
    CkSpider::setCkCacheDir(spider, "c:/spiderCache/")
    CkSpider::setCkFetchFromCache(spider, 1)
    CkSpider::setCkUpdateCache(spider, 1)

    While CkStringArray::ckCount(seedUrls) > 0

        url.s = CkStringArray::ckPop(seedUrls)
        CkSpider::ckInitialize(spider,url)

        ; Spider 5 URLs of this domain.
        ; but first, save the base domain in seenDomains
        domain.s = CkSpider::ckGetUrlDomain(spider,url)
        CkStringArray::ckAppend(seenDomains,CkSpider::ckGetBaseDomain(spider,domain))

        i.i

        For i = 0 To 4
            success = CkSpider::ckCrawlNext(spider)
            If success = 1

                ; Display the URL we just crawled.
                Debug CkSpider::ckLastUrl(spider)

                ; If the last URL was retrieved from cache,
                ; we won't wait.  Otherwise we'll wait 1 second
                ; before fetching the next URL.
                If CkSpider::ckLastFromCache(spider) <> 1
                    CkSpider::ckSleepMs(spider,1000)
                EndIf

            Else
                ; cause the loop to exit..
                i = 999
            EndIf

        Next

        ; Add the outbound links to seedUrls, except
        ; for the domains we've already seen.
        For i = 0 To CkSpider::ckNumOutboundLinks(spider) - 1

            url = CkSpider::ckGetOutboundLink(spider,i)
            domain.s = CkSpider::ckGetUrlDomain(spider,url)
            baseDomain.s = CkSpider::ckGetBaseDomain(spider,domain)
            If CkStringArray::ckContains(seenDomains,baseDomain) = 0
                ; Don't let our list of seedUrls grow too large.
                If CkStringArray::ckCount(seedUrls) < 1000
                    CkStringArray::ckAppend(seedUrls,url)
                EndIf

            EndIf

        Next

    Wend


    CkSpider::ckDispose(spider)
    CkStringArray::ckDispose(seenDomains)
    CkStringArray::ckDispose(seedUrls)


    ProcedureReturn
EndProcedure