crawler
Replace crawler configuration
Replaces the crawler configuration with a new one.
If you only want to change the crawler's name, you can use this operation. For other configuration changes, use the "Update configuration" endpoint instead instead, because changes made here aren't versioned.
When replacing the configuration, you must provide the full configuration, including any settings you want to keep.
patch/1/crawlers/{id}
Path parameters
idstring required
Universally unique identifier (UUID) of the crawler.
Example:e0f6db8a-24f5-4092-83a4-1b2c6cb6d809
Crawler ID.
Request body
Example request
{
"name": "test-crawler",
"config": {
"actions": [
{
"discoveryPatterns": [
"https://www.algolia.com/**"
],
"fileTypesToMatch": [
"html",
"pdf"
],
"hostnameAliases": {
"dev.example.com": "example.com"
},
"indexName": "algolia_website",
"pathAliases": {
"example.com": {
"/foo": "/bar"
}
},
"pathsToMatch": [
"https://www.algolia.com/**"
],
"selectorsToMatch": [
".products",
"!.featured"
]
}
],
"exclusionPatterns": [
"https://www.example.com/excluded",
"!https://www.example.com/this-one-url",
"https://www.example.com/exclude/**"
],
"externalData": [
"testCSV"
],
"ignoreQueryParams": [
"ref",
"utm_*"
],
"indexPrefix": "crawler_",
"linkExtractor": {
"source": "({ $, url, defaultExtractor }) => {\n if (/example.com\\/doc\\//.test(url.href)) {\n // For all pages under `/doc`, only extract the first found URL.\n return defaultExtractor().slice(0, 1)\n }\n // For all other pages, use the default.\n return defaultExtractor()\n}\n"
},
"login": {
"url": "https://example.com/secure/login-with-post",
"requestOptions": {
"method": "POST",
"headers": {
"Content-Type": "application/x-www-form-urlencoded"
},
"body": "id=my-id&password=my-password",
"timeout": 5000
}
},
"maxDepth": 5,
"maxUrls": 250,
"rateLimit": 4,
"renderJavaScript": {
"enabled": true,
"patterns": [
"http://www.mysite.com/dynamic-pages/**"
],
"waitTime": {
"min": 7000,
"max": 15000
}
},
"requestOptions": {
"headers": {
"Accept-Language": "fr-FR",
"Authorization": "Bearer Aerehdf==",
"Cookie": "session=1234"
}
},
"schedule": "every weekday at 12:00 pm",
"sitemaps": [
"https://example.com/sitemap.xyz"
],
"startUrls": [
"https://www.example.com"
]
}
}Response
OK
Example response
{
"taskId": "98458796-b7bb-4703-8b1b-785c1080b110"
}