(C#) A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Chilkat .NET Downloads Chilkat .NET Framework Chilkat for .NET Core

Chilkat.Spider spider = new Chilkat.Spider();

Chilkat.StringArray seenDomains = new Chilkat.StringArray();
Chilkat.StringArray seedUrls = new Chilkat.StringArray();

seenDomains.Unique = true;
seedUrls.Unique = true;

// You will need to change the start URL to something else...
seedUrls.Append("http://something.whateverYouWant.com/");

// Set outbound URL exclude patterns
// URLs matching any of these patterns will not be added to the 
// collection of outbound links.
spider.AddAvoidOutboundLinkPattern("*?id=*");
spider.AddAvoidOutboundLinkPattern("*.mypages.*");
spider.AddAvoidOutboundLinkPattern("*.personal.*");
spider.AddAvoidOutboundLinkPattern("*.comcast.*");
spider.AddAvoidOutboundLinkPattern("*.aol.*");
spider.AddAvoidOutboundLinkPattern("*~*");

// Use a cache so we don't have to re-fetch URLs previously fetched.
spider.CacheDir = "c:/spiderCache/";
spider.FetchFromCache = true;
spider.UpdateCache = true;

while (seedUrls.Count > 0) {

    string url = seedUrls.Pop();
    spider.Initialize(url);

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    string domain = spider.GetUrlDomain(url);
    seenDomains.Append(spider.GetBaseDomain(domain));

    int i;
    bool success;
    for (i = 0; i <= 4; i++) {
        success = spider.CrawlNext();
        if (success == true) {

            // Display the URL we just crawled.
            Debug.WriteLine(spider.LastUrl);

            // If the last URL was retrieved from cache,
            // we won't wait.  Otherwise we'll wait 1 second
            // before fetching the next URL.
            if (spider.LastFromCache != true) {
                spider.SleepMs(1000);
            }

        }
        else {
            // cause the loop to exit..
            i = 999;
        }

    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for (i = 0; i <= spider.NumOutboundLinks - 1; i++) {

        url = spider.GetOutboundLink(i);
        string domain = spider.GetUrlDomain(url);
        string baseDomain = spider.GetBaseDomain(domain);
        if (seenDomains.Contains(baseDomain) == false) {
            // Don't let our list of seedUrls grow too large.
            if (seedUrls.Count < 1000) {
                seedUrls.Append(url);
            }

        }

    }

}