Deep Crawl

import { BulkgridClient } from '@bulkgrid/sdk';

const client = new BulkgridClient({
  apiKey: process.env.BULKGRID_API_KEY ?? '',
  baseUrl: process.env.BULKGRID_BASE_URL ?? 'https://bulkgrid.com',
});

const data = await client.deepCrawl({
  url: 'https://example.com/docs',
  config: {
    maxDepth: 3,
    maxPages: 100,
    includePaths: ['/docs', '/blog'],
    excludePaths: ['/legal'],
    includeExternal: false,
    includeDocumentLinks: true,
    restrictToStartPath: true,
  },
  options: {
    formats: ['markdown', 'cleanHtml', 'links'],
    timeout: 30000,
    blockAds: true,
    useInteractions: true,
  },
});
console.log(data);

import os
from bulkgrid import BulkgridClient

with BulkgridClient(
    api_key=os.environ["BULKGRID_API_KEY"],
    base_url=os.environ.get("BULKGRID_BASE_URL", "https://bulkgrid.com"),
) as client:
    data = client.deep_crawl(
        {
            "url": "https://example.com/docs",
            "config": {
                "maxDepth": 3,
                "maxPages": 100,
                "includePaths": ["/docs", "/blog"],
                "excludePaths": ["/legal"],
                "includeExternal": False,
                "includeDocumentLinks": True,
                "restrictToStartPath": True,
            },
            "options": {
                "formats": ["markdown", "cleanHtml", "links"],
                "timeout": 30000,
                "blockAds": True,
                "useInteractions": True,
            },
        }
    )
    print(data)

curl "$BULKGRID_BASE_URL/api/v1/deep-crawl" \
  -H 'Content-Type: application/json' \
  -H "x-api-key: $BULKGRID_API_KEY" \
  -d '{
    "type": "deep_crawl",
    "url": "https://example.com/docs",
    "config": {
      "maxDepth": 3,
      "maxPages": 100,
      "includePaths": ["/docs", "/blog"],
      "excludePaths": ["/legal"],
      "includeExternal": false,
      "includeDocumentLinks": true,
      "restrictToStartPath": true
    },
    "options": {
      "formats": ["markdown", "cleanHtml", "links"],
      "timeout": 30000,
      "blockAds": true,
      "useInteractions": true
    }
  }'