(PowerShell) A Simple Web Crawler

This demonstrates a very simple web crawler using the Chilkat Spider component.

Chilkat .NET Downloads Chilkat .NET Assemblies

Add-Type -Path "C:\chilkat\ChilkatDotNet47-9.5.0-x64\ChilkatDotNet47.dll"

$spider = New-Object Chilkat.Spider

$seenDomains = New-Object Chilkat.StringArray
$seedUrls = New-Object Chilkat.StringArray

$seenDomains.Unique = $true
$seedUrls.Unique = $true

# You will need to change the start URL to something else...
$seedUrls.Append("http://something.whateverYouWant.com/")

# Set outbound URL exclude patterns
# URLs matching any of these patterns will not be added to the 
# collection of outbound links.
$spider.AddAvoidOutboundLinkPattern("*?id=*")
$spider.AddAvoidOutboundLinkPattern("*.mypages.*")
$spider.AddAvoidOutboundLinkPattern("*.personal.*")
$spider.AddAvoidOutboundLinkPattern("*.comcast.*")
$spider.AddAvoidOutboundLinkPattern("*.aol.*")
$spider.AddAvoidOutboundLinkPattern("*~*")

# Use a cache so we don't have to re-fetch URLs previously fetched.
$spider.CacheDir = "c:/spiderCache/"
$spider.FetchFromCache = $true
$spider.UpdateCache = $true

while ($seedUrls.Count -gt 0) {

    $url = $seedUrls.Pop()
    $spider.Initialize($url)

    # Spider 5 URLs of this domain.
    # but first, save the base domain in seenDomains
    $domain = $spider.GetUrlDomain($url)
    $seenDomains.Append($spider.GetBaseDomain($domain))

    for ($i = 0; $i -le 4; $i++) {
        $success = $spider.CrawlNext()
        if ($success -eq $true) {

            # Display the URL we just crawled.
            $($spider.LastUrl)

            # If the last URL was retrieved from cache,
            # we won't wait.  Otherwise we'll wait 1 second
            # before fetching the next URL.
            if ($spider.LastFromCache -ne $true) {
                $spider.SleepMs(1000)
            }

        }
        else {
            # cause the loop to exit..
            $i = 999
        }

    }

    # Add the outbound links to seedUrls, except
    # for the domains we've already seen.
    for ($i = 0; $i -le $spider.NumOutboundLinks - 1; $i++) {

        $url = $spider.GetOutboundLink($i)
        $domain = $spider.GetUrlDomain($url)
        $baseDomain = $spider.GetBaseDomain($domain)
        if ($seenDomains.Contains($baseDomain) -eq $false) {
            # Don't let our list of seedUrls grow too large.
            if ($seedUrls.Count -lt 1000) {
                $seedUrls.Append($url)
            }

        }

    }

}