(Node.js) A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Install Chilkat for Node.js and Electron using npm at Chilkat npm packages for Node.js Chilkat npm packages for Electron on Windows, Linux, MacOSX, and ARM

var os = require('os');
if (os.platform() == 'win32') {  
    if (os.arch() == 'ia32') {
        var chilkat = require('@chilkat/ck-node21-win-ia32');
    } else {
        var chilkat = require('@chilkat/ck-node21-win64'); 
    }
} else if (os.platform() == 'linux') {
    if (os.arch() == 'arm') {
        var chilkat = require('@chilkat/ck-node21-arm');
    } else if (os.arch() == 'x86') {
        var chilkat = require('@chilkat/ck-node21-linux32');
    } else {
        var chilkat = require('@chilkat/ck-node21-linux64');
    }
} else if (os.platform() == 'darwin') {
    if (os.arch() == 'arm64') {
        var chilkat = require('@chilkat/ck-node21-mac-m1');
    } else {
        var chilkat = require('@chilkat/ck-node21-macosx');
    }
}

function chilkatExample() {

    var spider = new chilkat.Spider();

    var seenDomains = new chilkat.StringArray();
    var seedUrls = new chilkat.StringArray();

    seenDomains.Unique = true;
    seedUrls.Unique = true;

    // You will need to change the start URL to something else...
    seedUrls.Append("http://something.whateverYouWant.com/");

    // Set outbound URL exclude patterns
    // URLs matching any of these patterns will not be added to the 
    // collection of outbound links.
    spider.AddAvoidOutboundLinkPattern("*?id=*");
    spider.AddAvoidOutboundLinkPattern("*.mypages.*");
    spider.AddAvoidOutboundLinkPattern("*.personal.*");
    spider.AddAvoidOutboundLinkPattern("*.comcast.*");
    spider.AddAvoidOutboundLinkPattern("*.aol.*");
    spider.AddAvoidOutboundLinkPattern("*~*");

    // Use a cache so we don't have to re-fetch URLs previously fetched.
    spider.CacheDir = "c:/spiderCache/";
    spider.FetchFromCache = true;
    spider.UpdateCache = true;

    while (seedUrls.Count > 0) {

        var url = seedUrls.Pop();
        spider.Initialize(url);

        // Spider 5 URLs of this domain.
        // but first, save the base domain in seenDomains
        var domain = spider.GetUrlDomain(url);
        seenDomains.Append(spider.GetBaseDomain(domain));

        var i;
        var success;
        for (i = 0; i <= 4; i++) {
            success = spider.CrawlNext();
            if (success == true) {

                // Display the URL we just crawled.
                console.log(spider.LastUrl);

                // If the last URL was retrieved from cache,
                // we won't wait.  Otherwise we'll wait 1 second
                // before fetching the next URL.
                if (spider.LastFromCache !== true) {
                    spider.SleepMs(1000);
                }

            }
            else {
                // cause the loop to exit..
                i = 999;
            }

        }

        // Add the outbound links to seedUrls, except
        // for the domains we've already seen.
        for (i = 0; i <= spider.NumOutboundLinks - 1; i++) {

            url = spider.GetOutboundLink(i);
            var domain = spider.GetUrlDomain(url);
            var baseDomain = spider.GetBaseDomain(domain);
            if (seenDomains.Contains(baseDomain) == false) {
                // Don't let our list of seedUrls grow too large.
                if (seedUrls.Count < 1000) {
                    seedUrls.Append(url);
                }

            }

        }

    }


}

chilkatExample();