SQL Server

Crawling the Web

If the Chilkat Spider component only crawls a single site, how do you crawl the Web? The answer is simple: as you crawl a site, the spider collects outbound links and makes them accessible to you. You may then instantiate an instance of the Spider object for each site, and crawl it. The task of keeping track of what sites you've already crawled is left to you. This example retrieves the home page of http://www.joelonsoftware.com/ and displays the outbound links. SQL Server

-- Important: See this note about string length limitations for strings returned by sp_OAMethod calls.
--
CREATE PROCEDURE ChilkatSample
AS
BEGIN
    DECLARE @hr int
    DECLARE @iTmp0 int
    -- Important: Do not use nvarchar(max).  See the warning about using nvarchar(max).
    DECLARE @sTmp0 nvarchar(4000)
    DECLARE @success int
    SELECT @success = 0

    DECLARE @spider int
    EXEC @hr = sp_OACreate 'Chilkat.Spider', @spider OUT
    IF @hr <> 0
    BEGIN
        PRINT 'Failed to create ActiveX component'
        RETURN
    END

    -- The Initialize method may be called with just the domain name,
    -- such as "www.joelonsoftware.com" or a full URL.  If you pass only
    -- the domain name, you must add URLs to the unspidered list by calling
    -- AddUnspidered.  Otherwise, the URL you pass to Initialize is the 1st
    -- URL in the unspidered list.
    EXEC sp_OAMethod @spider, 'Initialize', NULL, 'www.joelonsoftware.com'

    EXEC sp_OAMethod @spider, 'AddUnspidered', NULL, 'http://www.joelonsoftware.com/'

    EXEC sp_OAMethod @spider, 'CrawlNext', @success OUT

    DECLARE @i int

    EXEC sp_OAGetProperty @spider, 'NumOutboundLinks', @iTmp0 OUT
    SELECT @i = 0
    WHILE @i <= @iTmp0 - 1
      BEGIN
        EXEC sp_OAMethod @spider, 'GetOutboundLink', @sTmp0 OUT, @i
        PRINT @sTmp0
        SELECT @i = @i + 1
      END

    EXEC @hr = sp_OADestroy @spider


END
GO