v49

latestOpenAPI 3.0.2raw.githubusercontent.com2026-02-0620117129.6 KB

crawler

Retrieve a configuration version

Retrieves the specified version of the crawler configuration.

You can use this to restore a previous version of the configuration.

get/1/crawlers/{id}/config/versions/{version}

Path parameters

idstring required

Universally unique identifier (UUID) of the crawler.

Example:e0f6db8a-24f5-4092-83a4-1b2c6cb6d809

Crawler ID.

versioninteger required

This crawler's version nmber.

Response

versioninteger required

Version of the configuration. Version 1 is the initial configuration you used when creating the crawler.

createdAtstring required

Date and time when the object was created, in RFC 3339 format.

authorIdstring required

Universally unique identifier (UUID) of the user who created this version of the configuration.

Example response

{
  "config": {
    "actions": [
      {
        "discoveryPatterns": [
          "https://www.algolia.com/**"
        ],
        "fileTypesToMatch": [
          "html",
          "pdf"
        ],
        "hostnameAliases": {
          "dev.example.com": "example.com"
        },
        "indexName": "algolia_website",
        "pathAliases": {
          "example.com": {
            "/foo": "/bar"
          }
        },
        "pathsToMatch": [
          "https://www.algolia.com/**"
        ],
        "selectorsToMatch": [
          ".products",
          "!.featured"
        ]
      }
    ],
    "exclusionPatterns": [
      "https://www.example.com/excluded",
      "!https://www.example.com/this-one-url",
      "https://www.example.com/exclude/**"
    ],
    "externalData": [
      "testCSV"
    ],
    "ignoreQueryParams": [
      "ref",
      "utm_*"
    ],
    "indexPrefix": "crawler_",
    "linkExtractor": {
      "source": "({ $, url, defaultExtractor }) => {\n  if (/example.com\\/doc\\//.test(url.href)) {\n    // For all pages under `/doc`, only extract the first found URL.\n    return defaultExtractor().slice(0, 1)\n  }\n  // For all other pages, use the default.\n  return defaultExtractor()\n}\n"
    },
    "login": {
      "url": "https://example.com/secure/login-with-post",
      "requestOptions": {
        "method": "POST",
        "headers": {
          "Content-Type": "application/x-www-form-urlencoded"
        },
        "body": "id=my-id&password=my-password",
        "timeout": 5000
      }
    },
    "maxDepth": 5,
    "maxUrls": 250,
    "rateLimit": 4,
    "renderJavaScript": {
      "enabled": true,
      "patterns": [
        "http://www.mysite.com/dynamic-pages/**"
      ],
      "waitTime": {
        "min": 7000,
        "max": 15000
      }
    },
    "requestOptions": {
      "headers": {
        "Accept-Language": "fr-FR",
        "Authorization": "Bearer Aerehdf==",
        "Cookie": "session=1234"
      }
    },
    "schedule": "every weekday at 12:00 pm",
    "sitemaps": [
      "https://example.com/sitemap.xyz"
    ],
    "startUrls": [
      "https://www.example.com"
    ]
  },
  "createdAt": "2023-07-04T12:49:15Z",
  "authorId": "7d79f0dd-2dab-4296-8098-957a1fdc0637"
}