Objective-C

A Simple Web Crawler

See more Spider Examples

This demonstrates a very simple web crawler using the Chilkat Spider component. Objective-C

#import <CkoSpider.h>
#import <CkoStringArray.h>
#import <NSString.h>

BOOL success = NO;

CkoSpider *spider = [[CkoSpider alloc] init];

CkoStringArray *seenDomains = [[CkoStringArray alloc] init];
CkoStringArray *seedUrls = [[CkoStringArray alloc] init];

seenDomains.Unique = YES;
seedUrls.Unique = YES;

// You will need to change the start URL to something else...
[seedUrls Append: @"http://something.whateverYouWant.com/"];

// Set outbound URL exclude patterns
// URLs matching any of these patterns will not be added to the 
// collection of outbound links.
[spider AddAvoidOutboundLinkPattern: @"*?id=*"];
[spider AddAvoidOutboundLinkPattern: @"*.mypages.*"];
[spider AddAvoidOutboundLinkPattern: @"*.personal.*"];
[spider AddAvoidOutboundLinkPattern: @"*.comcast.*"];
[spider AddAvoidOutboundLinkPattern: @"*.aol.*"];
[spider AddAvoidOutboundLinkPattern: @"*~*"];

// Use a cache so we don't have to re-fetch URLs previously fetched.
spider.CacheDir = @"c:/spiderCache/";
spider.FetchFromCache = YES;
spider.UpdateCache = YES;

while ([seedUrls.Count intValue] > 0) {

    NSString *url = [seedUrls Pop];
    [spider Initialize: url];

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    NSString *domain = [spider GetUrlDomain: url];
    [seenDomains Append: [spider GetBaseDomain: domain]];

    int i;

    for (i = 0; i <= 4; i++) {
        success = [spider CrawlNext];
        if (success == YES) {

            // Display the URL we just crawled.
            NSLog(@"%@",spider.LastUrl);

            // If the last URL was retrieved from cache,
            // we won't wait.  Otherwise we'll wait 1 second
            // before fetching the next URL.
            if (spider.LastFromCache != YES) {
                [spider SleepMs: [NSNumber numberWithInt: 1000]];
            }

        }
        else {
            // cause the loop to exit..
            i = 999;
        }

    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for (i = 0; i <= [spider.NumOutboundLinks intValue] - 1; i++) {

        url = [spider GetOutboundLink: [NSNumber numberWithInt: i]];
        NSString *domain = [spider GetUrlDomain: url];
        NSString *baseDomain = [spider GetBaseDomain: domain];
        if ([seenDomains Contains: baseDomain] == NO) {
            // Don't let our list of seedUrls grow too large.
            if ([seedUrls.Count intValue] < 1000) {
                [seedUrls Append: url];
            }

        }

    }

}