crawler
Test crawl a URL
Tests a URL with the crawler's configuration and shows the extracted records.
You can test configuration changes by overriding specific parts before updating the full configuration.
post/1/crawlers/{id}/test
Path parameters
idstring required
Universally unique identifier (UUID) of the crawler.
Example:e0f6db8a-24f5-4092-83a4-1b2c6cb6d809
Crawler ID.
Request body
Example request
{
"url": "https://www.algolia.com/blog",
"config": {
"actions": [
{
"discoveryPatterns": [
"https://www.algolia.com/**"
],
"fileTypesToMatch": [
"html",
"pdf"
],
"hostnameAliases": {
"dev.example.com": "example.com"
},
"indexName": "algolia_website",
"pathAliases": {
"example.com": {
"/foo": "/bar"
}
},
"pathsToMatch": [
"https://www.algolia.com/**"
],
"selectorsToMatch": [
".products",
"!.featured"
]
}
],
"exclusionPatterns": [
"https://www.example.com/excluded",
"!https://www.example.com/this-one-url",
"https://www.example.com/exclude/**"
],
"externalData": [
"testCSV"
],
"ignoreQueryParams": [
"ref",
"utm_*"
],
"indexPrefix": "crawler_",
"linkExtractor": {
"source": "({ $, url, defaultExtractor }) => {\n if (/example.com\\/doc\\//.test(url.href)) {\n // For all pages under `/doc`, only extract the first found URL.\n return defaultExtractor().slice(0, 1)\n }\n // For all other pages, use the default.\n return defaultExtractor()\n}\n"
},
"login": {
"url": "https://example.com/secure/login-with-post",
"requestOptions": {
"method": "POST",
"headers": {
"Content-Type": "application/x-www-form-urlencoded"
},
"body": "id=my-id&password=my-password",
"timeout": 5000
}
},
"maxDepth": 5,
"maxUrls": 250,
"rateLimit": 4,
"renderJavaScript": {
"enabled": true,
"patterns": [
"http://www.mysite.com/dynamic-pages/**"
],
"waitTime": {
"min": 7000,
"max": 15000
}
},
"requestOptions": {
"headers": {
"Accept-Language": "fr-FR",
"Authorization": "Bearer Aerehdf==",
"Cookie": "session=1234"
}
},
"schedule": "every weekday at 12:00 pm",
"sitemaps": [
"https://example.com/sitemap.xyz"
],
"startUrls": [
"https://www.example.com"
]
}
}Response
OK
Example response
{
"startDate": "2024-04-02T15:34:29Z",
"endDate": "2024-04-02T15:34:29Z",
"logs": [
[
"Processing url 'https://www.algolia.com/blog'"
]
],
"records": [
{
"indexName": "testIndex",
"records": [
{
"objectID": "https://www.algolia.com/blog",
"numberOfLinks": 2
}
],
"recordsPerExtractor": [
{
"index": 0,
"type": "custom",
"records": [
{
"objectID": "https://www.algolia.com/blog"
}
]
}
]
}
],
"links": [
"https://www.algolia.com/blog/engineering/challenging-migration-heroku-google-kubernetes-engine",
"https://www.algolia.com/blog/engineering/a-tale-of-two-engines-algolia-unity"
],
"externalData": {
"externalData1": {
"data1": "val1",
"data2": "val2"
},
"externalData2": {
"data1": "val1",
"data2": "val2"
}
},
"error": {}
}