Create Crawl

Example request

{
  "exclude_paths": [
    "/exclude-this-path",
    "/and-this-path"
  ],
  "extract_options": {
    "auto_driver_configuration": {
      "vx10": 2,
      "vx10-pro": 0,
      "vx6-fast": 1,
      "vx6-stealth": 1,
      "vx8": 5,
      "vx8-pro": 5
    },
    "body": {
      "key": "value"
    },
    "browser": "chrome",
    "browser_actions": [
      {
        "goto": "https://example.com/login"
      },
      {
        "wait_for_element": "#login-form"
      },
      {
        "fill": {
          "selector": "#username",
          "value": "user@example.com"
        }
      },
      {
        "fill": {
          "selector": "#password",
          "value": "password123"
        }
      },
      {
        "click": "#submit"
      },
      {
        "screenshot": {
          "full_page": true
        }
      }
    ],
    "city": "Los Angeles",
    "cookies": "sessionId=abc123; userId=user456",
    "device": "desktop",
    "driver": "vx8",
    "expected_status_codes": [
      200,
      201
    ],
    "formats": [
      "html"
    ],
    "headers": {
      "Accept-Language": "en-US",
      "User-Agent": "CustomBot/1.0"
    },
    "http2": true,
    "is_xhr": true,
    "markdown_backend": "full_page",
    "method": "GET",
    "network_capture": [
      {
        "resource_type": "document"
      }
    ],
    "os": "windows",
    "parse": true,
    "request_timeout": 30000,
    "skill": "dynamic-content",
    "state": "CA",
    "tag": "campaign-2024-q1"
  },
  "include_paths": [
    "/include-this-path",
    "/and-this-path"
  ],
  "limit": 100,
  "max_discovery_depth": 3,
  "name": "The best crawl ever",
  "sitemap": "include"
}