Node.js

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Node.js

NODEJS_PRELUDE

function chilkatExample() {

    var success = false;

    var spider = new chilkat.Spider();

    var seenDomains = new chilkat.StringArray();
    var seedUrls = new chilkat.StringArray();

    seenDomains.Unique = true;
    seedUrls.Unique = true;

    //  You will need to change the start URL to something else...
    seedUrls.Append("http://something.whateverYouWant.com/");

    //  Set outbound URL exclude patterns
    //  URLs matching any of these patterns will not be added to the 
    //  collection of outbound links.
    spider.AddAvoidOutboundLinkPattern("*?id=*");
    spider.AddAvoidOutboundLinkPattern("*.mypages.*");
    spider.AddAvoidOutboundLinkPattern("*.personal.*");
    spider.AddAvoidOutboundLinkPattern("*.comcast.*");
    spider.AddAvoidOutboundLinkPattern("*.aol.*");
    spider.AddAvoidOutboundLinkPattern("*~*");

    //  Use a cache so we don't have to re-fetch URLs previously fetched.
    spider.CacheDir = "c:/spiderCache/";
    spider.FetchFromCache = true;
    spider.UpdateCache = true;

    while (seedUrls.Count > 0) {

        var url = seedUrls.Pop();
        spider.Initialize(url);

        //  Spider 5 URLs of this domain.
        //  but first, save the base domain in seenDomains
        var domain = spider.GetUrlDomain(url);
        seenDomains.Append(spider.GetBaseDomain(domain));

        var i;

        for (i = 0; i <= 4; i++) {
            success = spider.CrawlNext();
            if (success == true) {

                //  Display the URL we just crawled.
                console.log(spider.LastUrl);

                //  If the last URL was retrieved from cache,
                //  we won't wait.  Otherwise we'll wait 1 second
                //  before fetching the next URL.
                if (spider.LastFromCache !== true) {
                    spider.SleepMs(1000);
                }

            }
            else {
                //  cause the loop to exit..
                i = 999;
            }

        }

        //  Add the outbound links to seedUrls, except
        //  for the domains we've already seen.
        for (i = 0; i <= spider.NumOutboundLinks - 1; i++) {

            url = spider.GetOutboundLink(i);
            var domain = spider.GetUrlDomain(url);
            var baseDomain = spider.GetBaseDomain(domain);
            if (seenDomains.Contains(baseDomain) == false) {
                //  Don't let our list of seedUrls grow too large.
                if (seedUrls.Count < 1000) {
                    seedUrls.Append(url);
                }

            }

        }

    }


}

chilkatExample();