curl -X POST https://scrapeforllm.com/api/app/scrapes \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer YOUR_API_KEY" \
  -d '{
    "url": "https://docs.example.com",
    "type": "crawl"
  }'

const response = await fetch("https://scrapeforllm.com/api/app/scrapes", {
  method: "POST",
  headers: {
    "Content-Type": "application/json",
    Authorization: "Bearer YOUR_API_KEY",
  },
  body: JSON.stringify({
    url: "https://docs.example.com",
    type: "crawl",
  }),
});

const { scrape } = await response.json();
console.log("Crawl started:", scrape.id);

import requests

response = requests.post(
    "https://scrapeforllm.com/api/app/scrapes",
    headers={
        "Content-Type": "application/json",
        "Authorization": "Bearer YOUR_API_KEY",
    },
    json={
        "url": "https://docs.example.com",
        "type": "crawl",
    },
)

scrape = response.json()["scrape"]
print("Crawl started:", scrape["id"])

# Poll every 2 seconds until done
curl https://scrapeforllm.com/api/app/scrapes/YOUR_SCRAPE_ID \
  -H "Authorization: Bearer YOUR_API_KEY"

async function pollCrawl(scrapeId) {
  while (true) {
    const res = await fetch(
      `https://scrapeforllm.com/api/app/scrapes/${scrapeId}`,
      { headers: { Authorization: "Bearer YOUR_API_KEY" } }
    );
    const data = await res.json();

    if (data.scrape.status === "completed") {
      console.log(`Done! ${data.scrape.pagesScraped} pages scraped.`);
      return data.scrape;
    }

    if (data.scrape.status === "failed") {
      throw new Error("Crawl failed");
    }

    // Show progress
    if (data.progress) {
      console.log(`Progress: ${data.progress.completed}/${data.progress.total}`);
    }

    await new Promise((r) => setTimeout(r, 2000));

}
}

const result = await pollCrawl("YOUR_SCRAPE_ID");

import time
import requests

def poll_crawl(scrape_id):
    while True:
        res = requests.get(
            f"https://scrapeforllm.com/api/app/scrapes/{scrape_id}",
            headers={"Authorization": "Bearer YOUR_API_KEY"},
        )
        data = res.json()

        if data["scrape"]["status"] == "completed":
            print(f"Done! {data['scrape']['pagesScraped']} pages scraped.")
            return data["scrape"]

        if data["scrape"]["status"] == "failed":
            raise Exception("Crawl failed")

        # Show progress
        if "progress" in data:
            print(f"Progress: {data['progress']['completed']}/{data['progress']['total']}")

        time.sleep(2)

result = poll_crawl("YOUR_SCRAPE_ID")