Crawl

import { BulkgridClient } from '@bulkgrid/sdk';

const client = new BulkgridClient({
  apiKey: process.env.BULKGRID_API_KEY ?? '',
  baseUrl: process.env.BULKGRID_BASE_URL ?? 'https://bulkgrid.com',
});

const run = await client.crawl({
  urls: ['https://example.com/docs', 'https://example.com/pricing'],
  strategy: 'lexical',
  options: {
    formats: ['markdown', 'cleanHtml', 'links'],
    timeout: 30000,
    blockAds: true,
    useInteractions: true,
    waitForImages: false,
  },
});

import os
from bulkgrid import BulkgridClient

with BulkgridClient(
    api_key=os.environ["BULKGRID_API_KEY"],
    base_url=os.environ.get("BULKGRID_BASE_URL", "https://bulkgrid.com"),
) as client:
    data = client.crawl(
        {
            "urls": ["https://example.com/docs", "https://example.com/pricing"],
            "strategy": "lexical",
            "options": {
                "formats": ["markdown", "cleanHtml", "links"],
                "timeout": 30000,
                "blockAds": True,
                "useInteractions": True,
                "waitForImages": False,
            },
        }
    )
    print(data)

curl "$BULKGRID_BASE_URL/api/v1/crawl" \
  -H 'Content-Type: application/json' \
  -H "x-api-key: $BULKGRID_API_KEY" \
  -d '{
    "type": "crawl",
    "urls": [
      "https://example.com/docs",
      "https://example.com/pricing"
    ],
    "strategy": "lexical",
    "options": {
      "formats": ["markdown", "cleanHtml", "links"],
      "timeout": 30000,
      "blockAds": true,
      "useInteractions": true,
      "waitForImages": false
    }
  }'