v1

latestOpenAPI 3.0.02026-07-263039205.4 KB

Create a Crawl

Create and start a job to spider and extract pages through a site.

post/crawl

Response

200

responsestring

Example response

{
  "response": "Successfully added urls for spidering.",
  "jobs": [
    {
      "maxToProcessPerSubdomain": -1,
      "jobStatus": {
        "message": "Job is initializing."
      },
      "maxHops": -1,
      "onlyProcessIfNew": 1,
      "downloadJson": "https://api.diffbot.com/v3/crawl/download/<REDACTED>.json",
      "maxRounds": -1,
      "restrictDomain": 1,
      "type": "crawl",
      "apiUrl": "https://api.diffbot.com/v3/analyze",
      "useCanonical": 1,
      "maxToCrawlPerSubdomain": -1,
      "jobCreationTimeUTC": 1726163287,
      "downloadUrls": "https://api.diffbot.com/v3/crawl/download/<REDACTED>.csv",
      "obeyRobots": 1,
      "maxToProcess": 100000,
      "seedRecrawlFrequency": -1,
      "seeds": "https://example.com",
      "crawlDelay": 0.25,
      "currentTime": 1726163288,
      "maxToCrawl": 100000,
      "currentTimeUTC": 1726163288,
      "name": "test-crawl"
    }
  ]
}