Java

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Chilkat2 Java

import com.chilkatsoft.chilkat2.Spider;
import com.chilkatsoft.chilkat2.StringArray;

public class ChilkatExample {

    public static void main(String[] args) {
        boolean success = false;

        Spider spider = new Spider();

        StringArray seenDomains = new StringArray();
        StringArray seedUrls = new StringArray();

        seenDomains.setUnique(true);
        seedUrls.setUnique(true);

        // You will need to change the start URL to something else...
        seedUrls.append("http://something.whateverYouWant.com/");

        // Set outbound URL exclude patterns
        // URLs matching any of these patterns will not be added to the
        // collection of outbound links.
        spider.addAvoidOutboundLinkPattern("*?id=*");
        spider.addAvoidOutboundLinkPattern("*.mypages.*");
        spider.addAvoidOutboundLinkPattern("*.personal.*");
        spider.addAvoidOutboundLinkPattern("*.comcast.*");
        spider.addAvoidOutboundLinkPattern("*.aol.*");
        spider.addAvoidOutboundLinkPattern("*~*");

        // Use a cache so we don't have to re-fetch URLs previously fetched.
        spider.setCacheDir("c:/spiderCache/");
        spider.setFetchFromCache(true);
        spider.setUpdateCache(true);

        while (seedUrls.getCount() > 0) {
            String url = seedUrls.pop();
            spider.initialize(url);

            // Spider 5 URLs of this domain.
            // but first, save the base domain in seenDomains
            String domain = spider.getUrlDomain(url);
            seenDomains.append(spider.getBaseDomain(domain));

            int numCrawled = 0;
            success = true;
            while ((success) && (numCrawled < 5)) {
                success = spider.crawlNext();
                if (success) {
                    // Display the URL we just crawled.
                    System.out.println(spider.getLastUrl());

                    // If the last URL was retrieved from cache,
                    // we won't wait.  Otherwise we'll wait 1 second
                    // before fetching the next URL.
                    if (!spider.getLastFromCache()) {
                        spider.sleepMs(1000);
                    }

                    numCrawled = numCrawled + 1;
                }

                // If CrawlNext fails (no more URLs to crawl in this domain), success is false and the loop exits.
            }

            // Add the outbound links to seedUrls, except
            // for the domains we've already seen.
            for (int i = 0; i < spider.getNumOutboundLinks(); i++) {
                url = spider.getOutboundLink(i);
                domain = spider.getUrlDomain(url);
                String baseDomain = spider.getBaseDomain(domain);
                if (!seenDomains.contains(baseDomain)) {
                    // Don't let our list of seedUrls grow too large.
                    if (seedUrls.getCount() < 1000) {
                        seedUrls.append(url);
                    }
                }
            }
        }
    }
}