crawler
Create a crawler
Creates a new crawler with the provided configuration.
post/1/crawlers
Request body
Example request
{
"name": "test-crawler",
"config": {
"actions": [
{
"discoveryPatterns": [
"https://www.algolia.com/**"
],
"fileTypesToMatch": [
"html",
"pdf"
],
"hostnameAliases": {
"dev.example.com": "example.com"
},
"indexName": "algolia_website",
"pathAliases": {
"example.com": {
"/foo": "/bar"
}
},
"pathsToMatch": [
"https://www.algolia.com/**"
],
"selectorsToMatch": [
".products",
"!.featured"
]
}
],
"exclusionPatterns": [
"https://www.example.com/excluded",
"!https://www.example.com/this-one-url",
"https://www.example.com/exclude/**"
],
"externalData": [
"testCSV"
],
"ignoreQueryParams": [
"ref",
"utm_*"
],
"indexPrefix": "crawler_",
"linkExtractor": {
"source": "({ $, url, defaultExtractor }) => {\n if (/example.com\\/doc\\//.test(url.href)) {\n // For all pages under `/doc`, only extract the first found URL.\n return defaultExtractor().slice(0, 1)\n }\n // For all other pages, use the default.\n return defaultExtractor()\n}\n"
},
"login": {
"url": "https://example.com/secure/login-with-post",
"requestOptions": {
"method": "POST",
"headers": {
"Content-Type": "application/x-www-form-urlencoded"
},
"body": "id=my-id&password=my-password",
"timeout": 5000
}
},
"maxDepth": 5,
"maxUrls": 250,
"rateLimit": 4,
"renderJavaScript": {
"enabled": true,
"patterns": [
"http://www.mysite.com/dynamic-pages/**"
],
"waitTime": {
"min": 7000,
"max": 15000
}
},
"requestOptions": {
"headers": {
"Accept-Language": "fr-FR",
"Authorization": "Bearer Aerehdf==",
"Cookie": "session=1234"
}
},
"schedule": "every weekday at 12:00 pm",
"sitemaps": [
"https://example.com/sitemap.xyz"
],
"startUrls": [
"https://www.example.com"
]
}
}Response
OK
Example response
{
"id": "e0f6db8a-24f5-4092-83a4-1b2c6cb6d809"
}