(PHP Extension) A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Chilkat PHP Downloads PHP Extension for Windows, Linux, MacOS, Alpine Linux, Solaris

<?php

// The version number (9_5_0) should match version of the Chilkat extension used, omitting the micro-version number.
// For example, if using Chilkat v9.5.0.48, then include as shown here:
include("chilkat_9_5_0.php");

$spider = new CkSpider();

$seenDomains = new CkStringArray();
$seedUrls = new CkStringArray();

$seenDomains->put_Unique(true);
$seedUrls->put_Unique(true);

// You will need to change the start URL to something else...
$seedUrls->Append('http://something.whateverYouWant.com/');

// Set outbound URL exclude patterns
// URLs matching any of these patterns will not be added to the 
// collection of outbound links.
$spider->AddAvoidOutboundLinkPattern('*?id=*');
$spider->AddAvoidOutboundLinkPattern('*.mypages.*');
$spider->AddAvoidOutboundLinkPattern('*.personal.*');
$spider->AddAvoidOutboundLinkPattern('*.comcast.*');
$spider->AddAvoidOutboundLinkPattern('*.aol.*');
$spider->AddAvoidOutboundLinkPattern('*~*');

// Use a cache so we don't have to re-fetch URLs previously fetched.
$spider->put_CacheDir('c:/spiderCache/');
$spider->put_FetchFromCache(true);
$spider->put_UpdateCache(true);

while ($seedUrls->get_Count() > 0) {

    $url = $seedUrls->pop();
    $spider->Initialize($url);

    // Spider 5 URLs of this domain.
    // but first, save the base domain in seenDomains
    $domain = $spider->getUrlDomain($url);
    $seenDomains->Append($spider->getBaseDomain($domain));

    for ($i = 0; $i <= 4; $i++) {
        $success = $spider->CrawlNext();
        if ($success == true) {

            // Display the URL we just crawled.
            print $spider->lastUrl() . "\n";

            // If the last URL was retrieved from cache,
            // we won't wait.  Otherwise we'll wait 1 second
            // before fetching the next URL.
            if ($spider->get_LastFromCache() != true) {
                $spider->SleepMs(1000);
            }

        }
        else {
            // cause the loop to exit..
            $i = 999;
        }

    }

    // Add the outbound links to seedUrls, except
    // for the domains we've already seen.
    for ($i = 0; $i <= $spider->get_NumOutboundLinks() - 1; $i++) {

        $url = $spider->getOutboundLink($i);
        $domain = $spider->getUrlDomain($url);
        $baseDomain = $spider->getBaseDomain($domain);
        if ($seenDomains->Contains($baseDomain) == false) {
            // Don't let our list of seedUrls grow too large.
            if ($seedUrls->get_Count() < 1000) {
                $seedUrls->Append($url);
            }

        }

    }

}


?>