Xbase++

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Xbase++

LOCAL nSuccess
LOCAL oSpider
LOCAL oSeenDomains
LOCAL oSeedUrls
LOCAL cUrl
LOCAL cDomain
LOCAL i
LOCAL cDomain
LOCAL cBaseDomain

nSuccess := 0

oSpider := CreateObject("Chilkat.Spider")

oSeenDomains := CreateObject("Chilkat.StringArray")
oSeedUrls := CreateObject("Chilkat.StringArray")

oSeenDomains:Unique := 1
oSeedUrls:Unique := 1

//  You will need to change the start URL to something else...
oSeedUrls:Append("http://something.whateverYouWant.com/")

//  Set outbound URL exclude patterns
//  URLs matching any of these patterns will not be added to the 
//  collection of outbound links.
oSpider:AddAvoidOutboundLinkPattern("*?id=*")
oSpider:AddAvoidOutboundLinkPattern("*.mypages.*")
oSpider:AddAvoidOutboundLinkPattern("*.personal.*")
oSpider:AddAvoidOutboundLinkPattern("*.comcast.*")
oSpider:AddAvoidOutboundLinkPattern("*.aol.*")
oSpider:AddAvoidOutboundLinkPattern("*~*")

//  Use a cache so we don't have to re-fetch URLs previously fetched.
oSpider:CacheDir := "c:/spiderCache/"
oSpider:FetchFromCache := 1
oSpider:UpdateCache := 1

DO WHILE oSeedUrls:Count > 0

    cUrl := oSeedUrls:Pop()
    oSpider:Initialize(cUrl)

    //  Spider 5 URLs of this domain.
    //  but first, save the base domain in seenDomains
    cDomain := oSpider:GetUrlDomain(cUrl)
    oSeenDomains:Append(oSpider:GetBaseDomain(cDomain))

    FOR i := 0 TO 4
        nSuccess := oSpider:CrawlNext()
        IF (nSuccess == 1)

            //  Display the URL we just crawled.
            ? oSpider:LastUrl

            //  If the last URL was retrieved from cache,
            //  we won't wait.  Otherwise we'll wait 1 second
            //  before fetching the next URL.
            IF (oSpider:LastFromCache != 1)
                oSpider:SleepMs(1000)
            ENDIF

        ELSE
            //  cause the loop to exit..
            i := 999
        ENDIF

    NEXT

    //  Add the outbound links to seedUrls, except
    //  for the domains we've already seen.
    FOR i := 0 TO oSpider:NumOutboundLinks - 1

        cUrl := oSpider:GetOutboundLink(i)
        cDomain := oSpider:GetUrlDomain(cUrl)
        cBaseDomain := oSpider:GetBaseDomain(cDomain)
        IF (oSeenDomains:Contains(cBaseDomain) == 0)
            //  Don't let our list of seedUrls grow too large.
            IF (oSeedUrls:Count < 1000)
                oSeedUrls:Append(cUrl)
            ENDIF

        ENDIF

    NEXT

ENDDO

oSpider:destroy()
oSeenDomains:destroy()
oSeedUrls:destroy()