(Node.js) Fetch robots.txt for a Site

The Chilkat Spider library is robots.txt compliant. It automatically fetches a site's robots.txt file and adheres to it. It will not download pages denied by robots.txt. Pages excluded by robots.txt will not appear in the Spider's "unspidered" list. This example shows how to explicitly download and review the robots.txt for a given site.

Install Chilkat for Node.js and Electron using npm at Chilkat npm packages for Node.js Chilkat npm packages for Electron on Windows, Linux, MacOS

var os = require('os');
if (os.platform() == 'win32') {  
  var chilkat = require('@chilkat/ck-node23-win64'); 
} else if (os.platform() == 'linux') {
    if (os.arch() == 'arm') {
        var chilkat = require('@chilkat/ck-node23-linux-arm');
    } else if (os.arch() == 'arm64') {
        var chilkat = require('@chilkat/ck-node23-linux-arm64');
    } else {
        var chilkat = require('@chilkat/ck-node23-linux-x64');
    }
} else if (os.platform() == 'darwin') {
  var chilkat = require('@chilkat/ck-node23-mac-universal');
}

function chilkatExample() {

    var spider = new chilkat.Spider();

    spider.Initialize("www.chilkatsoft.com");

    var robotsText;
    robotsText = spider.FetchRobotsText();

    console.log(robotsText);

}

chilkatExample();