React Native

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. React Native

import { Spider, StringArray } from '@chilkat/react-native'

async function chilkatExample() {
  let success = false;

  const spider = new Spider();

  const seenDomains = new StringArray();
  const seedUrls = new StringArray();

  seenDomains.unique = true;
  seedUrls.unique = true;

  // You will need to change the start URL to something else...
  seedUrls.append('http://something.whateverYouWant.com/');

  // Set outbound URL exclude patterns
  // URLs matching any of these patterns will not be added to the 
  // collection of outbound links.
  spider.addAvoidOutboundLinkPattern('*?id=*');
  spider.addAvoidOutboundLinkPattern('*.mypages.*');
  spider.addAvoidOutboundLinkPattern('*.personal.*');
  spider.addAvoidOutboundLinkPattern('*.comcast.*');
  spider.addAvoidOutboundLinkPattern('*.aol.*');
  spider.addAvoidOutboundLinkPattern('*~*');

  // Use a cache so we don't have to re-fetch URLs previously fetched.
  spider.cacheDir = 'c:/spiderCache/';
  spider.fetchFromCache = true;
  spider.updateCache = true;

  while (seedUrls.count > 0) {

    let url = seedUrls.pop();
    spider.initialize(url);

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    let domain = spider.getUrlDomain(url);
    seenDomains.append(spider.getBaseDomain(domain));

    let numCrawled = 0;
    success = true;
    while ((success) && (numCrawled < 5)) {
      success = true;
      try {
        await spider.crawlNextAsync();
      } catch {
        success = false;
      }
      if (success) {
        // Display the URL we just crawled.
        console.log(spider.lastUrl);

        // If the last URL was retrieved from cache,
        // we won't wait.  Otherwise we'll wait 1 second
        // before fetching the next URL.
        if (!spider.lastFromCache) {
          spider.sleepMs(1000);
        }

        numCrawled++;
      }

      // If CrawlNext fails (no more URLs to crawl in this domain), success is false and the loop exits.
    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for (let i = 0; i < spider.numOutboundLinks; i++) {

      url = spider.getOutboundLink(i);
      domain = spider.getUrlDomain(url);
      const baseDomain = spider.getBaseDomain(domain);
      if (!seenDomains.contains(baseDomain)) {
        // Don't let our list of seedUrls grow too large.
        if (seedUrls.count < 1000) {
          seedUrls.append(url);
        }
      }
    }
  }
}