CrewAI

pip install geonode-scraper-crewai python-dotenv

GEONODE_SCRAPER_API_KEY=your_geonode_key
SCRAPER_API_BASE_URL=https://scraper.geonode.io

import os
from dotenv import load_dotenv
from geonode_scraper_crewai import build_crewai_tools
from geonode_scraper_tools_core import ScraperToolSettings

load_dotenv()

settings = ScraperToolSettings(
    host=os.environ.get("SCRAPER_API_BASE_URL", "https://scraper.geonode.io"),
    api_key=os.environ["GEONODE_SCRAPER_API_KEY"],
)

tools = build_crewai_tools(settings=settings)
by_name = {tool.name: tool for tool in tools}

{
    "ok": True,
    "operation": "extract",
    "attempts": 1,
    "result": { ... },
}

response = by_name["scraper_extract_content"].run(
    url="https://docs.geonode.com/docs/scraper-api/quick-start",
    formats=["markdown"],
    processing_mode="sync",
)

result = response["result"]
markdown = (result.get("data") or {}).get("markdown") or ""
print("ok:", response["ok"])
print("tokens:", result.get("tokens_charged"))
print("markdown_len:", len(markdown))
print("preview:", markdown[:200])

ok: True
tokens: 1
markdown_len: 15871
preview: ---
canonical: https://docs.geonode.com/docs/scraper-api/quick-start
meta-description: Get your API key, authenticate requests, and choose the right API for your use case.
...

response = by_name["scraper_extract_content"].run(
    url="https://docs.geonode.com/docs/scraper-api/quick-start",
    formats=["markdown"],
    processing_mode="async",
)

result = response["result"]
print("job_id:", result["job_id"])
print("status:", result["status"])

job_id: e35c4d0e-9df3-43b9-9446-33c517b1cfc4
status: queued

response = by_name["scraper_wait_for_job"].run(
    job_id="e35c4d0e-9df3-43b9-9446-33c517b1cfc4",
    timeout_seconds=120,
)

result = response["result"]
markdown = (result.get("data") or {}).get("markdown") or ""
print("status:", result["status"])
print("markdown_len:", len(markdown))
print("poll_attempts:", response.get("poll_attempts"))

status: completed
markdown_len: 15871
poll_attempts: 4

response = by_name["scraper_get_job_result"].run(
    job_id="e35c4d0e-9df3-43b9-9446-33c517b1cfc4",
)

result = response["result"]
print("status:", result["status"])
print("tokens:", result.get("tokens_charged"))

status: completed
tokens: 1

response = by_name["scraper_list_jobs"].run(page=1, page_size=3)

result = response["result"]
print("page:", result["page"], "page_size:", result["page_size"])
for job in result.get("jobs") or []:
    print(job["job_id"], job["status"], job.get("url"))

page: 1 page_size: 3
752b8599-5915-441c-bc1f-b9fb0d938f72 completed ...
0bcd424e-b5ab-4b85-9cbd-5b44f7ed433c completed http://example.com/
0dbb126c-4331-4b4b-9d55-0c380ba69ae7 completed http://example.com/

response = by_name["scraper_create_batch"].run(
    urls=[
        "https://docs.geonode.com/docs/scraper-api/quick-start",
        "https://docs.geonode.com/docs/scraper-api",
    ],
    formats=["markdown"],
)

result = response["result"]
print("job_id:", result["job_id"])
print("accepted_urls:", result["accepted_urls"])
print("status:", result["status"])

job_id: 037be6d5-06e8-480f-9e22-4eac6cfb2966
accepted_urls: 2
status: queued

response = by_name["scraper_get_batch_status"].run(
    job_id="037be6d5-06e8-480f-9e22-4eac6cfb2966",
    page=1,
    page_size=10,
)

result = response["result"]
print(result["status"], result["completed_urls"], "/", result["total_urls"])

processing 0 / 2

response = by_name["scraper_wait_for_batch"].run(
    job_id="037be6d5-06e8-480f-9e22-4eac6cfb2966",
    timeout_seconds=120,
)

result = response["result"]
print(result["status"], result["completed_urls"], "/", result["total_urls"])
print("poll_attempts:", response.get("poll_attempts"))

completed 2 / 2
poll_attempts: 3

response = by_name["scraper_list_batch_jobs"].run(page=1, page_size=3)

for job in (response["result"].get("jobs") or []):
    print(
        job["job_id"],
        job["status"],
        job["completed_urls"],
        "/",
        job["accepted_urls"],
    )

037be6d5-06e8-480f-9e22-4eac6cfb2966 completed 2 / 2
600bb35a-d6fc-4e05-be49-816b8f3ad5d7 completed 2 / 2
55b11790-5c7f-4945-bc13-bd9a365a1835 completed 2 / 2

response = by_name["scraper_create_crawl"].run(
    url="https://docs.geonode.com/docs/scraper-api",
    depth=2,
    limit=3,
    formats=["markdown"],
    same_domain_only=True,
)

result = response["result"]
print("job_id:", result["job_id"])
print("estimated_pages:", result["estimated_pages"])
print("status:", result["status"])

job_id: dbc0fa1a-1fea-496c-abe4-a33c2b91efb7
estimated_pages: 3
status: queued

response = by_name["scraper_get_crawl_status"].run(
    job_id="dbc0fa1a-1fea-496c-abe4-a33c2b91efb7",
    page=1,
    page_size=10,
)

result = response["result"]
print(
    result["status"],
    result.get("completed_pages"),
    "/",
    result.get("total_pages"),
)

processing 0 / 1

response = by_name["scraper_wait_for_crawl"].run(
    job_id="dbc0fa1a-1fea-496c-abe4-a33c2b91efb7",
    timeout_seconds=180,
)

result = response["result"]
print(result["status"], result["completed_pages"], "/", result["total_pages"])
print("poll_attempts:", response.get("poll_attempts"))

completed 3 / 3
poll_attempts: 4

response = by_name["scraper_list_crawl_jobs"].run(page=1, page_size=2)

for job in (response["result"].get("jobs") or []):
    print(
        job["job_id"],
        job["status"],
        job["completed_pages"],
        "/",
        job["total_pages"],
    )

dbc0fa1a-1fea-496c-abe4-a33c2b91efb7 completed 3 / 3
ab83f113-5086-46d3-a045-3a42263b5c87 completed 3 / 3

response = by_name["scraper_map_urls"].run(
    url="https://docs.geonode.com/docs/scraper-api",
)

result = response["result"]
links = result.get("links") or []
print("link_count:", result.get("links_count") or len(links))
for link in links[:5]:
    print(link.get("source"), link.get("url"))

link_count: 112
sitemap https://docs.geonode.com/docs/scraper-api
sitemap https://docs.geonode.com/docs/scraper-api/quick-start
sitemap https://docs.geonode.com/docs/scraper-api/additional-resources/choosing_scraper_api_plan
sitemap https://docs.geonode.com/docs/scraper-api/additional-resources/faq
sitemap https://docs.geonode.com/docs/scraper-api/additional-resources/pricing-and-requests

response = by_name["scraper_list_map_jobs"].run(page=1, page_size=2)

for job in (response["result"].get("jobs") or []):
    print(job["job_id"], job["status"], job.get("links_count"), job.get("url"))

88b13fab-8d1b-421d-aa0e-256dd705b3aa completed 112 https://docs.geonode.com/docs/scraper-api
3abfc672-2aff-4daf-9f68-f033978ddfbd completed 112 https://docs.geonode.com/docs/scraper-api

response = by_name["scraper_get_map_job"].run(
    job_id="88b13fab-8d1b-421d-aa0e-256dd705b3aa",
)

result = response["result"]
print("status:", result["status"])
print("link_count:", result.get("links_count"))
for link in (result.get("links") or [])[:3]:
    print(link.get("source"), link.get("url"))

status: completed
link_count: 112
sitemap https://docs.geonode.com/docs/scraper-api
sitemap https://docs.geonode.com/docs/scraper-api/quick-start
sitemap https://docs.geonode.com/docs/scraper-api/additional-resources/choosing_scraper_api_plan

response = by_name["scraper_search"].run(query="geonode scraper api")

result = response["result"]
print("job_id:", result["job_id"])
print("hit_count:", result.get("results_count") or len(result.get("results") or []))
for hit in (result.get("results") or [])[:5]:
    print(hit["position"], hit["title"], hit["url"])

job_id: 7eeb9599-c39a-4eba-8e0d-2eaed5a8deab
hit_count: 15
1 Wholesale proxies & web data infrastructure | Geonode https://geonode.com/
2 Geonode - PyPI https://pypi.org/user/Geonode/
3 pavel.s - PyPI https://pypi.org/user/pavel.s/
4 Geonode Documentation | Geonode https://docs.geonode.com/
5 GeoNode https://geonode.org/

response = by_name["scraper_list_search_jobs"].run(page=1, page_size=2)

for job in (response["result"].get("jobs") or []):
    print(job["job_id"], job.get("query"), job["status"], job.get("results_count"))

7eeb9599-c39a-4eba-8e0d-2eaed5a8deab geonode scraper api completed 15
f80269e1-1911-4fd4-b1a1-fc0f7bbae7f9 geonode scraper api completed 15

response = by_name["scraper_get_search_job"].run(
    job_id="7eeb9599-c39a-4eba-8e0d-2eaed5a8deab",
)

result = response["result"]
print("status:", result["status"])
print("hit_count:", result.get("results_count"))
for hit in (result.get("results") or [])[:3]:
    print(hit["position"], hit["title"], hit["url"])

status: completed
hit_count: 15
1 Wholesale proxies & web data infrastructure | Geonode https://geonode.com/
2 Geonode - PyPI https://pypi.org/user/Geonode/
3 pavel.s - PyPI https://pypi.org/user/pavel.s/

response = by_name["scraper_get_statistics"].run()

result = response["result"]
print("extraction_count:", result.get("extraction_count"))
print("success_rate:", result.get("success_rate"))

extraction_count: 4401
success_rate: 0.95137

response = by_name["scraper_get_concurrency_usage"].run()

result = response["result"]
print(
    result["work_concurrency_in_use"],
    "/",
    result["work_concurrency_limit"],
)

0 / 50

response = by_name["scraper_check_health"].run()

result = response["result"]
print(result.get("service"), result.get("status"), result.get("version"))

Scraper API ok 0.1.0

tools = build_crewai_tools(
    settings=settings,
    operations=["extract", "map_urls", "create_crawl", "wait_for_crawl"],
)

print([tool.name for tool in tools])

['scraper_extract_content', 'scraper_map_urls', 'scraper_create_crawl', 'scraper_wait_for_crawl']

from crewai import Agent
from geonode_scraper_crewai import build_crewai_tools
from geonode_scraper_tools_core import ScraperToolSettings

settings = ScraperToolSettings(
    host="https://scraper.geonode.io",
    api_key=os.environ["GEONODE_SCRAPER_API_KEY"],
)

agent = Agent(
    role="Web Researcher",
    goal="Extract and inspect web content.",
    backstory="Focused on pulling structured data from URLs.",
    tools=build_crewai_tools(settings=settings),
)

from geonode_scraper_crewai import ScraperCrewAIToolkit
from geonode_scraper_tools_core import ScraperToolSettings

settings = ScraperToolSettings(
    host="https://scraper.geonode.io",
    api_key=os.environ["GEONODE_SCRAPER_API_KEY"],
)

toolkit = ScraperCrewAIToolkit.from_settings(settings)
tools = toolkit.get_tools()