Async Jobs & Polling

const API_BASE = "https://api.pocketutils.com/v1";
const API_KEY = "YOUR_API_KEY";
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));

function errorMessage(body, status) {
  return typeof body?.error === "string"
    ? body.error
    : body?.error?.message ?? `HTTP ${status}`;
}

async function crawlSite(tool, url, timeoutMs = 15 * 60 * 1000) {
  if (!["link-extract", "broken-links"].includes(tool)) {
    throw new Error("Choose link-extract or broken-links");
  }

  const deadline = Date.now() + timeoutMs;
  const endpoint = `${API_BASE}/${tool}/crawl`;
  const headers = { Authorization: `Bearer ${API_KEY}` };

  const started = await fetch(endpoint, {
    method: "POST",
    headers: { ...headers, "Content-Type": "application/json" },
    body: JSON.stringify({ url }),
    signal: AbortSignal.timeout(30000),
  });
  const startBody = await started.json();
  if (started.status !== 202 || startBody.success !== true ||
      !startBody.data?.jobId) {
    throw new Error(errorMessage(startBody, started.status));
  }

  const jobId = startBody.data.jobId;
  const statusUrl = `${endpoint}/${encodeURIComponent(jobId)}`;
  console.log("Started job:", jobId);
  let rateLimitRetries = 0;

  while (Date.now() < deadline) {
    const response = await fetch(statusUrl, {
      headers,
      cache: "no-store",
      signal: AbortSignal.timeout(Math.min(30000, deadline - Date.now())),
    });
    const body = await response.json();

    if (response.status === 429 && rateLimitRetries < 5) {
      rateLimitRetries += 1;
      const retryAfter = Number(response.headers.get("Retry-After"));
      const waitMs = Math.max(
        Number.isFinite(retryAfter) ? retryAfter * 1000 : 0,
        2000 * 2 ** rateLimitRetries,
      );
      await sleep(Math.min(waitMs, Math.max(0, deadline - Date.now())));
      continue;
    }

    if (!response.ok || body.success !== true) {
      throw new Error(errorMessage(body, response.status));
    }
    rateLimitRetries = 0;
    const job = body.data;
    console.log(job.status, job.progress);

    if (job.status === "done") return job.result;
    if (job.status === "error") {
      throw new Error(job.error ?? "Crawl failed");
    }

    await sleep(Math.min(2000, Math.max(0, deadline - Date.now())));
  }

  throw new Error(`Polling timed out for job ${jobId}; the server job may still be running`);
}

// Use "link-extract" for a link inventory or "broken-links" for a link audit.
crawlSite("broken-links", "https://example.com")
  .then((result) => console.log(JSON.stringify(result, null, 2)))
  .catch((error) => console.error(error.message));

import json
import time
from urllib.error import HTTPError
from urllib.parse import quote
from urllib.request import Request, urlopen

API_BASE = "https://api.pocketutils.com/v1"
API_KEY = "YOUR_API_KEY"


def request_json(url, method="GET", payload=None, timeout=30):
    headers = {"Authorization": f"Bearer {API_KEY}"}
    raw = None
    if payload is not None:
        headers["Content-Type"] = "application/json"
        raw = json.dumps(payload).encode("utf-8")
    else:
        headers["Cache-Control"] = "no-cache"
    request = Request(url, data=raw, headers=headers, method=method)
    try:
        response = urlopen(request, timeout=timeout)
    except HTTPError as error:
        response = error
    with response:
        return (
            response.code,
            dict(response.headers),
            json.loads(response.read().decode("utf-8")),
        )


def error_message(body, status):
    error = body.get("error")
    if isinstance(error, str):
        return error
    if isinstance(error, dict):
        return error.get("message", f"HTTP {status}")
    return f"HTTP {status}"


def crawl_site(tool, url, timeout_seconds=15 * 60):
    if tool not in ("link-extract", "broken-links"):
        raise ValueError("Choose link-extract or broken-links")

    deadline = time.monotonic() + timeout_seconds
    endpoint = f"{API_BASE}/{tool}/crawl"
    status, _, body = request_json(endpoint, "POST", {"url": url})
    job_id = body.get("data", {}).get("jobId")
    if status != 202 or body.get("success") is not True or not job_id:
        raise RuntimeError(error_message(body, status))

    print("Started job:", job_id)
    status_url = f"{endpoint}/{quote(job_id, safe='')}"
    rate_limit_retries = 0

    while time.monotonic() < deadline:
        remaining = deadline - time.monotonic()
        status, headers, body = request_json(
            status_url, timeout=max(0.1, min(30, remaining))
        )
        if status == 429 and rate_limit_retries < 5:
            rate_limit_retries += 1
            try:
                retry_after = float(headers.get("Retry-After", "0"))
            except ValueError:
                retry_after = 0
            wait = max(retry_after, 2 * 2 ** rate_limit_retries)
            time.sleep(min(wait, max(0, deadline - time.monotonic())))
            continue

        if status != 200 or body.get("success") is not True:
            raise RuntimeError(error_message(body, status))
        rate_limit_retries = 0
        job = body["data"]
        print(job["status"], job["progress"])
        if job["status"] == "done":
            return job["result"]
        if job["status"] == "error":
            raise RuntimeError(job.get("error", "Crawl failed"))
        time.sleep(min(2, max(0, deadline - time.monotonic())))

    raise TimeoutError(
        f"Polling timed out for job {job_id}; the server job may still be running"
    )


# Use "link-extract" for a link inventory or "broken-links" for a link audit.
result = crawl_site("broken-links", "https://example.com")
print(json.dumps(result, indent=2))