TypeScript SDK

npm install @geonode_scraper/scraper-sdk

import { Configuration } from "@geonode_scraper/scraper-sdk"

const configuration = new Configuration({ apiKey: process.env.GEONODE_SCRAPER_API_KEY })

import { ExtractionApi, OutputFormat } from "@geonode_scraper/scraper-sdk"

const api = new ExtractionApi(configuration)
const response = await api.extractV1ExtractPost({
  extractRequest: { url: "https://example.com", formats: [OutputFormat.Markdown] }
})

if ("data" in response) {
  console.log(response.data.markdown)
  console.log(response.tokensCharged) // what this request cost
}

import { OutputFormat, ProxyType, WaitUntil, type ExtractRequest } from "@geonode_scraper/scraper-sdk"

const request: ExtractRequest = {
  url: "https://example.com/products",
  formats: [OutputFormat.Markdown],
  renderJs: true, // use a headless browser
  proxy: { country: "DE", type: ProxyType.Residential },
  waitConfig: { waitUntil: WaitUntil.Networkidle, waitFor: "#content", waitTimeout: 2000 },
  timeout: 60000 // give up after 60 s
}

import { ExtractionApi, JobStatus, ProcessingMode } from "@geonode_scraper/scraper-sdk"

const api = new ExtractionApi(configuration)
const done: string[] = [JobStatus.Completed, JobStatus.Failed, JobStatus.Cancelled]

const accepted = await api.extractV1ExtractPost({
  extractRequest: { url: "https://example.com", processingMode: ProcessingMode.Async }
})
if ("jobId" in accepted) {
  let job = await api.getJobResultV1ExtractJobIdGet({ jobId: accepted.jobId })
  while (!done.includes(job.status)) {
    await new Promise((resolve) => setTimeout(resolve, 2000))
    job = await api.getJobResultV1ExtractJobIdGet({ jobId: accepted.jobId })
  }

  if (job.status === JobStatus.Completed) {
    console.log(job.data?.html)
  } else {
    console.log(job.status, job.error)
  }
}

import { BatchApi, OutputFormat } from "@geonode_scraper/scraper-sdk"

const api = new BatchApi(configuration)
const accepted = await api.createBatchV1BatchPost({
  batchRequest: { urls: ["https://example.com", "https://example.org"], formats: [OutputFormat.Markdown] }
})
const status = await api.getBatchStatusV1BatchJobIdGet({ jobId: accepted.jobId, page: 1, pageSize: 50 })
console.log(status.status, `${status.completedUrls}/${status.totalUrls}`)
for (const item of status.results ?? []) {
  console.log(item.url, item.status)
}

import { CrawlApi, OutputFormat } from "@geonode_scraper/scraper-sdk"

const api = new CrawlApi(configuration)
const accepted = await api.createCrawlV1CrawlPost({
  crawlRequest: { url: "https://example.com", depth: 2, limit: 50, formats: [OutputFormat.Markdown] }
})
console.log(accepted.jobId, accepted.estimatedPages)
const status = await api.getCrawlStatusV1CrawlJobIdGet({ jobId: accepted.jobId, page: 1, pageSize: 50 })
console.log(status.status, `${status.completedPages}/${status.totalPages}`)

import { MapApi } from "@geonode_scraper/scraper-sdk"

const result = await new MapApi(configuration).mapUrlsV1MapPost({ mapRequest: { url: "https://example.com" } })
for (const link of result.links ?? []) {
  console.log(link.url, link.source)
}

import { SearchApi } from "@geonode_scraper/scraper-sdk"

const result = await new SearchApi(configuration).searchV1SearchPost({ searchRequest: { query: "geonode scraper api" } })
for (const hit of result.results) {
  console.log(hit.position, hit.title, hit.url)
}

import { WebhookEventType, WebhooksApi } from "@geonode_scraper/scraper-sdk"

const webhook = await new WebhooksApi(configuration).createWebhookV1WebhooksPost({
  webhookCreate: { url: "https://your-app.example/hooks/geonode", eventType: WebhookEventType.ExtractCompleted }
})
console.log(webhook.id, webhook.secret) // store the secret: it signs every delivery

import { createHmac, timingSafeEqual } from "node:crypto"

export function isFromGeonode(rawBody: Buffer | string, signatureHeader: string, secret: string): boolean {
  const expected = Buffer.from(`sha256=${createHmac("sha256", secret).update(rawBody).digest("hex")}`)
  const received = Buffer.from(signatureHeader)
  return expected.length === received.length && timingSafeEqual(expected, received)
}

import { StatisticsApi, UsageApi } from "@geonode_scraper/scraper-sdk"

const usage = await new UsageApi(configuration).getConcurrencyUsageV1UsageConcurrencyGet()
console.log(`${usage.workConcurrencyInUse} of ${usage.workConcurrencyLimit} concurrent requests in use`)

const weekAgo = new Date(Date.now() - 7 * 24 * 3600 * 1000)
const stats = await new StatisticsApi(configuration).getStatisticsV1StatisticsGet({ startDate: weekAgo })
console.log(stats.extractionCount, stats.successRate, stats.periodTokensUsed)

import { ApiResponseError, ExtractionApi } from "@geonode_scraper/scraper-sdk"

try {
  await new ExtractionApi(configuration).extractV1ExtractPost({ extractRequest: { url: "https://example.com" } })
} catch (error) {
  if (error instanceof ApiResponseError) {
    console.log(error.status, error.body)
    if (error.status === 429) {
      console.log("retry after", error.response.headers.get("Retry-After"), "seconds")
    }
  } else {
    throw error
  }
}