Chilkat HOME .NET Core C# Android™ AutoIt C C# C++ Chilkat2-Python CkPython Classic ASP DataFlex Delphi ActiveX Delphi DLL Go Java Lianja Mono C# Node.js Objective-C PHP ActiveX PHP Extension Perl PowerBuilder PowerShell PureBasic Ruby SQL Server Swift 2 Swift 3,4,5... Tcl Unicode C Unicode C++ VB.NET VBScript Visual Basic 6.0 Visual FoxPro Xojo Plugin
(Unicode C) Avoid URLs Matching Any of a Set of PatternsDemonstrates how to use "avoid patterns" to prevent spidering any URL that matches a wildcarded pattern. This example avoids URLs containing the substrings "java", "python", or "perl".
#include <C_CkSpiderW.h> void ChilkatSample(void) { HCkSpiderW spider; int i; BOOL success; spider = CkSpiderW_Create(); // The spider object crawls a single web site at a time. As you'll see // in later examples, you can collect outbound links and use them to // crawl the web. For now, we'll simply spider 10 pages of chilkatsoft.com CkSpiderW_Initialize(spider,L"www.chilkatsoft.com"); // Add the 1st URL: CkSpiderW_AddUnspidered(spider,L"http://www.chilkatsoft.com/"); // Avoid URLs matching these patterns: CkSpiderW_AddAvoidPattern(spider,L"*java*"); CkSpiderW_AddAvoidPattern(spider,L"*python*"); CkSpiderW_AddAvoidPattern(spider,L"*perl*"); // Begin crawling the site by calling CrawlNext repeatedly. for (i = 0; i <= 9; i++) { success = CkSpiderW_CrawlNext(spider); if (success == TRUE) { // Show the URL of the page just spidered. wprintf(L"%s\n",CkSpiderW_lastUrl(spider)); // The HTML is available in the LastHtml property } else { // Did we get an error or are there no more URLs to crawl? if (CkSpiderW_getNumUnspidered(spider) == 0) { wprintf(L"No more URLs to spider\n"); } else { wprintf(L"%s\n",CkSpiderW_lastErrorText(spider)); } } // Sleep 1 second before spidering the next URL. CkSpiderW_SleepMs(spider,1000); } CkSpiderW_Dispose(spider); } |
© 2000-2024 Chilkat Software, Inc. All Rights Reserved.