From fa2df18bf3f35549d0da14583c3581359c5cb15d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 11:54:18 +0200 Subject: [PATCH 01/11] docs: restructure documentation to align with SDK and Client docs - Add a Concepts section holding the component and abstraction pages moved from Guides, plus a new Logging page. - Merge the Examples section into Concepts and Guides: each example is folded into its related page or combined into a new guide (Crawling links, Stopping and resuming crawlers), and redundant duplicates are dropped. - Number the section directories and files to drive sidebar order, matching the apify-sdk-python and apify-client-python docs conventions. - Enable @docusaurus/plugin-client-redirects with version-aware redirects for every moved URL. - Update the navbar, footer, homepage, README, and lint config paths, and add frontmatter descriptions to pages that lacked them. --- README.md | 2 +- .../beautifulsoup_crawler_example.py | 0 .../code_examples/parsel_crawler_example.py | 0 .../playwright_crawler_example.py | 0 .../playwright_crawler_headful_example.py | 0 .../{quick-start => 01_quick-start}/index.mdx | 7 +- .../01_setting_up.mdx | 1 + .../02_first_crawler.mdx | 1 + .../03_adding_more_urls.mdx | 1 + .../04_real_world_project.mdx | 1 + .../05_crawling.mdx | 1 + .../06_scraping.mdx | 1 + .../07_saving_data.mdx | 18 +- .../08_refactoring.mdx | 3 +- .../09_running_in_cloud.mdx | 2 +- .../code_examples/02_bs.py | 0 .../code_examples/02_bs_better.py | 0 .../code_examples/02_request_queue.py | 0 .../code_examples/03_enqueue_strategy.py | 0 .../code_examples/03_finding_new_links.py | 0 .../code_examples/03_globs.py | 0 .../code_examples/03_original_code.py | 0 .../code_examples/03_transform_request.py | 0 .../code_examples/04_sanity_check.py | 0 .../code_examples/05_crawling_detail.py | 0 .../code_examples/05_crawling_listing.py | 0 .../code_examples/06_scraping.py | 0 .../code_examples/07_final_code.py | 0 .../code_examples/07_first_code.py | 0 .../code_examples/08_main.py | 0 .../code_examples/08_routes.py | 0 .../code_examples/09_apify_sdk.py | 0 .../code_examples}/__init__.py | 0 .../code_examples/routes.py | 0 .../index.mdx | 1 + .../01_architecture_overview.mdx} | 33 ++-- .../02_http_crawlers.mdx} | 27 ++- .../03_playwright_crawler.mdx} | 39 ++++- .../04_playwright_crawler_adaptive.mdx} | 9 + .../05_request_router.mdx} | 7 +- .../06_request_loaders.mdx} | 18 +- .../07_storages.mdx} | 45 +++-- .../08_storage_clients.mdx} | 5 - .../09_http_clients.mdx} | 7 +- .../10_session_management.mdx} | 0 .../11_cookie_management.mdx} | 4 +- .../12_http_headers.mdx} | 11 +- .../13_proxy_management.mdx} | 2 +- .../14_scaling_crawlers.mdx} | 2 +- .../15_request_throttling.mdx} | 0 .../16_error_handling.mdx} | 27 ++- docs/03_concepts/17_logging.mdx | 70 ++++++++ .../18_service_locator.mdx} | 7 +- .../cookie_management/initial_cookies.py | 0 .../cookie_management/persist_cookies.py | 0 .../cookie_management/playwright_cookies.py | 0 .../cookie_management/read_write_cookies.py | 0 .../cookie_management/retry_pinned_session.py | 0 .../retry_restore_cookies.py | 0 .../cookie_management/retry_single_session.py | 0 .../change_handle_error_status.py | 0 .../error_handling/disable_retry.py | 0 .../error_handling/handle_proxy_error.py | 0 .../parsel_crawler_with_error_snapshotter.py | 0 ...aywright_crawler_with_error_snapshotter.py | 0 .../parsel_curl_impersonate_example.py | 0 .../http_clients/parsel_httpx_example.py | 0 .../http_clients/parsel_impit_example.py | 0 .../code_examples/http_crawlers}/__init__.py | 0 .../http_crawlers/beautifulsoup_example.py | 0 .../http_crawlers/custom_crawler_example.py | 0 .../http_crawlers}/file_download.py | 0 .../http_crawlers}/file_download_stream.py | 0 .../http_crawlers/http_example.py | 0 .../http_crawlers/lexbor_parser.py | 0 .../http_crawlers/lxml_parser.py | 0 .../http_crawlers/lxml_saxonche_parser.py | 0 .../http_crawlers/parsel_example.py | 0 .../http_crawlers/pyquery_parser.py | 0 .../http_crawlers/scrapling_parser.py | 0 .../http_crawlers/selectolax_adaptive_run.py | 0 .../http_crawlers/selectolax_context.py | 0 .../http_crawlers/selectolax_crawler.py | 0 .../http_crawlers/selectolax_crawler_run.py | 0 .../http_crawlers/selectolax_parser.py | 0 .../http_headers/browser_page_headers.py | 0 .../code_examples/http_headers/set_headers.py | 0 .../logging}/configure_json_logging.py | 0 .../browser_configuration_example.py | 0 .../browser_pool_launch_hooks_example.py | 0 .../browser_pool_page_hooks_example.py | 0 .../capture_screenshot_using_playwright.py | 0 .../multiple_launch_example.py | 0 .../navigation_hooks_example.py | 0 .../playwright_block_requests.py | 0 .../playwright_crawler}/playwright_crawler.py | 0 .../plugin_browser_configuration_example.py | 0 .../adaptive_playwright_crawler.py | 0 .../playwright_crawler_adaptive/handler.py | 0 .../init_beautifulsoup.py | 0 .../init_parsel.py | 0 .../init_prediction.py | 0 .../pre_nav_hooks.py | 0 .../proxy_management/inspecting_bs_example.py | 0 .../proxy_management/inspecting_pw_example.py | 0 .../integration_bs_example.py | 0 .../integration_pw_example.py | 0 .../proxy_management/quick_start_example.py | 0 .../proxy_management/session_bs_example.py | 0 .../proxy_management/session_pw_example.py | 0 .../proxy_management/tiers_bs_example.py | 0 .../proxy_management/tiers_pw_example.py | 0 .../request_loaders/rl_basic_example.py | 0 .../rl_basic_example_with_persist.py | 0 .../request_loaders/rl_tandem_example.py | 0 .../rl_tandem_example_explicit.py | 0 .../request_loaders/sitemap_basic_example.py | 0 .../sitemap_example_with_persist.py | 0 .../request_loaders/sitemap_tandem_example.py | 0 .../sitemap_tandem_example_explicit.py | 0 .../using_sitemap_request_loader.py | 0 .../adaptive_crawler_handlers.py | 0 .../request_router/basic_request_handlers.py | 0 .../custom_router_default_only.py | 0 .../request_router/error_handler.py | 0 .../request_router/failed_request_handler.py | 0 .../request_router/http_pre_navigation.py | 0 .../playwright_pre_navigation.py | 0 .../request_router/router_middleware.py | 0 .../request_router/simple_default_handler.py | 0 .../request_throttling/throttling_example.py | 0 .../max_tasks_per_minute_example.py | 0 .../min_and_max_concurrency_example.py | 0 .../service_locator/service_conflicts.py | 0 .../service_crawler_configuration.py | 0 .../service_crawler_event_manager.py | 0 .../service_crawler_storage_client.py | 0 .../service_locator_configuration.py | 0 .../service_locator_event_manager.py | 0 .../service_locator_storage_client.py | 0 .../service_storage_configuration.py | 0 .../service_storage_storage_client.py | 0 .../session_management/multi_sessions_http.py | 0 .../session_management/one_session_http.py | 0 .../session_management/sm_basic.py | 0 .../session_management/sm_beautifulsoup.py | 0 .../session_management/sm_http.py | 0 .../session_management/sm_parsel.py | 0 .../session_management/sm_playwright.py | 0 .../session_management/sm_standalone.py | 0 .../custom_storage_client_example.py | 0 ...ile_system_storage_client_basic_example.py | 0 ...em_storage_client_configuration_example.py | 0 .../memory_storage_client_basic_example.py | 0 .../redis_storage_client_basic_example.py | 0 ...is_storage_client_configuration_example.py | 0 .../registering_storage_clients_example.py | 0 .../sql_storage_client_basic_example.py | 0 ...ql_storage_client_configuration_example.py | 0 .../storages/cleaning_do_not_purge_example.py | 0 .../cleaning_purge_explicitly_example.py | 0 .../storages/dataset_basic_example.py | 0 .../storages/dataset_with_crawler_example.py | 0 .../dataset_with_crawler_explicit_example.py | 0 .../export_entire_dataset_to_file_csv.py | 0 .../export_entire_dataset_to_file_json.py | 0 .../storages/helper_add_requests_example.py | 0 .../storages/helper_enqueue_links_example.py | 0 .../storages/kvs_basic_example.py | 0 .../storages/kvs_with_crawler_example.py | 0 .../kvs_with_crawler_explicit_example.py | 0 .../code_examples/storages/opening.py | 0 .../storages/rq_basic_example.py | 0 .../storages/rq_with_crawler_example.py | 0 .../rq_with_crawler_explicit_example.py | 0 docs/04_guides/01_crawling_links.mdx | 141 ++++++++++++++++ .../02_fill_and_submit_web_form.mdx} | 9 +- .../03_crawler_login.mdx} | 35 +++- .../04_avoid_blocking.mdx} | 16 +- .../05_respect_robots_txt_file.mdx} | 7 +- docs/04_guides/06_stopping_and_resuming.mdx | 56 +++++++ .../07_run_parallel_crawlers.mdx} | 9 +- .../08_running_in_web_server.mdx} | 2 +- .../09_trace_and_monitor_crawlers.mdx} | 0 .../10_creating_web_archive.mdx} | 2 +- .../11_secure_scraping.mdx} | 4 +- .../12_extending_crawlee.mdx} | 8 +- .../13_pydantic_ai_crawler.mdx} | 0 .../14_stagehand_crawler.mdx} | 2 +- .../15_scrapy_migration.mdx} | 18 +- ...default_fingerprint_generator_with_args.py | 0 .../playwright_crawler_with_camoufox.py | 0 .../playwright_with_cloakbrowser.py | 0 .../playwright_with_fingerprint_generator.py | 0 .../crawl_all_links_on_website_bs.py | 0 .../crawl_all_links_on_website_pw.py | 0 .../crawling_links}/crawl_multiple_urls_bs.py | 0 .../crawling_links}/crawl_multiple_urls_pw.py | 0 .../crawl_specific_links_on_website_bs.py | 0 .../crawl_specific_links_on_website_pw.py | 0 ...l_website_with_relative_links_all_links.py | 0 ...website_with_relative_links_same_domain.py | 0 ...bsite_with_relative_links_same_hostname.py | 0 ...website_with_relative_links_same_origin.py | 0 ...ct_and_add_specific_links_on_website_bs.py | 0 ...ct_and_add_specific_links_on_website_pw.py | 0 .../manual_archiving_parsel_crawler.py | 0 .../manual_archiving_playwright_crawler.py | 0 .../simple_pw_through_proxy_pywb_server.py | 0 .../fill_and_submit_web_form_automated.py | 0 .../fill_and_submit_web_form_crawler.py | 0 .../fill_and_submit_web_form_request.py | 0 .../code_examples/login_crawler/http_login.py | 0 .../login_crawler/playwright_login.py | 0 .../using_browser_profiles_chrome.py | 0 .../using_browser_profiles_firefox.py | 0 .../additional_instructions_example.py | 0 .../pydantic_ai_crawler/basic_example.py | 0 .../custom_distiller_example.py | 0 .../pydantic_ai_crawler/debugging_example.py | 0 .../selector_extractor_example.py | 0 .../usage_limit_example.py | 0 .../respect_robots_on_skipped_request.py | 0 .../respect_robots_txt_file.py | 0 .../run_parallel_crawlers.py | 0 .../running_in_web_server}/__init__.py | 0 .../running_in_web_server/crawler.py | 0 .../running_in_web_server/server.py | 0 .../scrapy_migration/crawlee_concurrency.py | 0 .../scrapy_migration/crawlee_crawlspider.py | 0 .../crawlee_error_handling.py | 0 .../scrapy_migration/crawlee_export.py | 0 .../scrapy_migration/crawlee_labels.py | 0 .../scrapy_migration/crawlee_playwright.py | 0 .../scrapy_migration/crawlee_post.py | 0 .../scrapy_migration/crawlee_proxy.py | 0 .../scrapy_migration/crawlee_quotes.py | 0 .../scrapy_migration/crawlee_throttling.py | 0 .../scrapy_migration/crawlee_user_data.py | 0 .../scrapy_migration/scrapy_authors.py | 0 .../scrapy_migration/scrapy_cb_kwargs.py | 0 .../scrapy_migration/scrapy_concurrency.py | 0 .../scrapy_migration/scrapy_crawlspider.py | 0 .../scrapy_migration/scrapy_errback.py | 0 .../scrapy_migration/scrapy_export.py | 0 .../scrapy_migration/scrapy_formrequest.py | 0 .../scrapy_migration/scrapy_playwright.py | 0 .../scrapy_playwright_settings.py | 0 .../scrapy_migration/scrapy_proxy.py | 0 .../scrapy_migration/scrapy_quotes.py | 0 .../scrapy_migration/scrapy_throttling.py | 0 .../stagehand_crawler/basic_example.py | 0 .../stagehand_crawler/browserbase_example.py | 0 .../beautifulsoup_crawler_keep_alive.py | 0 .../beautifulsoup_crawler_stop.py | 0 .../resuming_paused_crawl.py | 0 .../instrument_crawler.py | 0 .../apify_platform.mdx | 4 +- .../aws_lambda.mdx | 2 +- .../apify/crawler_as_actor_example.py | 0 .../code_examples/apify/get_public_url.py | 0 .../apify/log_with_config_example.py | 0 .../apify/proxy_advanced_example.py | 0 .../code_examples/apify/proxy_example.py | 0 .../aws/beautifulsoup_crawler_lambda.py | 0 .../aws/playwright_crawler_lambda.py | 0 .../code_examples/aws/playwright_dockerfile | 0 .../code_examples/google/cloud_run_example.py | 0 .../code_examples/google/google_example.py | 0 .../google_cloud.mdx | 2 +- .../google_cloud_run.mdx | 0 .../upgrading_to_v0x.md | 1 + .../upgrading_to_v1.md | 7 +- docs/examples/add_data_to_dataset.mdx | 40 ----- docs/examples/beautifulsoup_crawler.mdx | 15 -- .../capture_screenshot_using_playwright.mdx | 19 --- ..._page_snapshots_with_error_snapshotter.mdx | 27 --- .../code_examples/add_data_to_dataset_bs.py | 35 ---- .../add_data_to_dataset_dataset.py | 15 -- .../code_examples/add_data_to_dataset_pw.py | 35 ---- .../code_examples/beautifulsoup_crawler.py | 57 ------- docs/examples/code_examples/parsel_crawler.py | 47 ------ ...ight_crawler_with_fingerprint_generator.py | 44 ----- docs/examples/crawl_all_links_on_website.mdx | 33 ---- docs/examples/crawl_multiple_urls.mdx | 27 --- .../crawl_specific_links_on_website.mdx | 47 ------ .../crawl_website_with_relative_links.mdx | 52 ------ docs/examples/crawler_keep_alive.mdx | 15 -- docs/examples/crawler_stop.mdx | 15 -- .../export_entire_dataset_to_file.mdx | 47 ------ docs/examples/file_download.mdx | 25 --- docs/examples/json_logging.mdx | 57 ------- docs/examples/parsel_crawler.mdx | 15 -- docs/examples/playwright_crawler.mdx | 19 --- docs/examples/playwright_crawler_adaptive.mdx | 20 --- ...playwright_crawler_with_block_requests.mdx | 27 --- .../playwright_crawler_with_camoufox.mdx | 26 --- ...ght_crawler_with_fingerprint_generator.mdx | 17 -- docs/examples/resuming_paused_crawl.mdx | 35 ---- docs/examples/using_browser_profile.mdx | 39 ----- .../examples/using_sitemap_request_loader.mdx | 22 --- pyproject.toml | 10 +- website/docusaurus.config.js | 154 +++++++++++++----- website/sidebars.js | 58 +++---- website/src/pages/index.js | 10 +- 305 files changed, 670 insertions(+), 1111 deletions(-) rename docs/{quick-start => 01_quick-start}/code_examples/beautifulsoup_crawler_example.py (100%) rename docs/{quick-start => 01_quick-start}/code_examples/parsel_crawler_example.py (100%) rename docs/{quick-start => 01_quick-start}/code_examples/playwright_crawler_example.py (100%) rename docs/{quick-start => 01_quick-start}/code_examples/playwright_crawler_headful_example.py (100%) rename docs/{quick-start => 01_quick-start}/index.mdx (92%) rename docs/{introduction => 02_introduction}/01_setting_up.mdx (98%) rename docs/{introduction => 02_introduction}/02_first_crawler.mdx (98%) rename docs/{introduction => 02_introduction}/03_adding_more_urls.mdx (98%) rename docs/{introduction => 02_introduction}/04_real_world_project.mdx (98%) rename docs/{introduction => 02_introduction}/05_crawling.mdx (96%) rename docs/{introduction => 02_introduction}/06_scraping.mdx (98%) rename docs/{introduction => 02_introduction}/07_saving_data.mdx (86%) rename docs/{introduction => 02_introduction}/08_refactoring.mdx (94%) rename docs/{introduction => 02_introduction}/09_running_in_cloud.mdx (98%) rename docs/{introduction => 02_introduction}/code_examples/02_bs.py (100%) rename docs/{introduction => 02_introduction}/code_examples/02_bs_better.py (100%) rename docs/{introduction => 02_introduction}/code_examples/02_request_queue.py (100%) rename docs/{introduction => 02_introduction}/code_examples/03_enqueue_strategy.py (100%) rename docs/{introduction => 02_introduction}/code_examples/03_finding_new_links.py (100%) rename docs/{introduction => 02_introduction}/code_examples/03_globs.py (100%) rename docs/{introduction => 02_introduction}/code_examples/03_original_code.py (100%) rename docs/{introduction => 02_introduction}/code_examples/03_transform_request.py (100%) rename docs/{introduction => 02_introduction}/code_examples/04_sanity_check.py (100%) rename docs/{introduction => 02_introduction}/code_examples/05_crawling_detail.py (100%) rename docs/{introduction => 02_introduction}/code_examples/05_crawling_listing.py (100%) rename docs/{introduction => 02_introduction}/code_examples/06_scraping.py (100%) rename docs/{introduction => 02_introduction}/code_examples/07_final_code.py (100%) rename docs/{introduction => 02_introduction}/code_examples/07_first_code.py (100%) rename docs/{introduction => 02_introduction}/code_examples/08_main.py (100%) rename docs/{introduction => 02_introduction}/code_examples/08_routes.py (100%) rename docs/{introduction => 02_introduction}/code_examples/09_apify_sdk.py (100%) rename docs/{guides/code_examples/http_crawlers => 02_introduction/code_examples}/__init__.py (100%) rename docs/{introduction => 02_introduction}/code_examples/routes.py (100%) rename docs/{introduction => 02_introduction}/index.mdx (94%) rename docs/{guides/architecture_overview.mdx => 03_concepts/01_architecture_overview.mdx} (91%) rename docs/{guides/http_crawlers.mdx => 03_concepts/02_http_crawlers.mdx} (85%) rename docs/{guides/playwright_crawler.mdx => 03_concepts/03_playwright_crawler.mdx} (75%) rename docs/{guides/playwright_crawler_adaptive.mdx => 03_concepts/04_playwright_crawler_adaptive.mdx} (95%) rename docs/{guides/request_router.mdx => 03_concepts/05_request_router.mdx} (89%) rename docs/{guides/request_loaders.mdx => 03_concepts/06_request_loaders.mdx} (92%) rename docs/{guides/storages.mdx => 03_concepts/07_storages.mdx} (84%) rename docs/{guides/storage_clients.mdx => 03_concepts/08_storage_clients.mdx} (96%) rename docs/{guides/http_clients.mdx => 03_concepts/09_http_clients.mdx} (89%) rename docs/{guides/session_management.mdx => 03_concepts/10_session_management.mdx} (100%) rename docs/{guides/cookie_management.mdx => 03_concepts/11_cookie_management.mdx} (96%) rename docs/{guides/http_headers.mdx => 03_concepts/12_http_headers.mdx} (91%) rename docs/{guides/proxy_management.mdx => 03_concepts/13_proxy_management.mdx} (98%) rename docs/{guides/scaling_crawlers.mdx => 03_concepts/14_scaling_crawlers.mdx} (95%) rename docs/{guides/request_throttling.mdx => 03_concepts/15_request_throttling.mdx} (100%) rename docs/{guides/error_handling.mdx => 03_concepts/16_error_handling.mdx} (56%) create mode 100644 docs/03_concepts/17_logging.mdx rename docs/{guides/service_locator.mdx => 03_concepts/18_service_locator.mdx} (91%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/initial_cookies.py (100%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/persist_cookies.py (100%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/playwright_cookies.py (100%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/read_write_cookies.py (100%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/retry_pinned_session.py (100%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/retry_restore_cookies.py (100%) rename docs/{guides => 03_concepts}/code_examples/cookie_management/retry_single_session.py (100%) rename docs/{guides => 03_concepts}/code_examples/error_handling/change_handle_error_status.py (100%) rename docs/{guides => 03_concepts}/code_examples/error_handling/disable_retry.py (100%) rename docs/{guides => 03_concepts}/code_examples/error_handling/handle_proxy_error.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/error_handling}/parsel_crawler_with_error_snapshotter.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/error_handling}/playwright_crawler_with_error_snapshotter.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_clients/parsel_curl_impersonate_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_clients/parsel_httpx_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_clients/parsel_impit_example.py (100%) rename docs/{guides/code_examples/running_in_web_server => 03_concepts/code_examples/http_crawlers}/__init__.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/beautifulsoup_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/custom_crawler_example.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/http_crawlers}/file_download.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/http_crawlers}/file_download_stream.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/http_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/lexbor_parser.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/lxml_parser.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/lxml_saxonche_parser.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/parsel_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/pyquery_parser.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/scrapling_parser.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/selectolax_adaptive_run.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/selectolax_context.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/selectolax_crawler.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/selectolax_crawler_run.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_crawlers/selectolax_parser.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_headers/browser_page_headers.py (100%) rename docs/{guides => 03_concepts}/code_examples/http_headers/set_headers.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/logging}/configure_json_logging.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler/browser_configuration_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler/browser_pool_page_hooks_example.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/playwright_crawler}/capture_screenshot_using_playwright.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler/multiple_launch_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler/navigation_hooks_example.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/playwright_crawler}/playwright_block_requests.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/playwright_crawler}/playwright_crawler.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler/plugin_browser_configuration_example.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/playwright_crawler_adaptive}/adaptive_playwright_crawler.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler_adaptive/handler.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler_adaptive/init_parsel.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler_adaptive/init_prediction.py (100%) rename docs/{guides => 03_concepts}/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/inspecting_bs_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/inspecting_pw_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/integration_bs_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/integration_pw_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/quick_start_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/session_bs_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/session_pw_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/tiers_bs_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/proxy_management/tiers_pw_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/rl_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/rl_basic_example_with_persist.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/rl_tandem_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/rl_tandem_example_explicit.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/sitemap_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/sitemap_example_with_persist.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/sitemap_tandem_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_loaders/sitemap_tandem_example_explicit.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/request_loaders}/using_sitemap_request_loader.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/adaptive_crawler_handlers.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/basic_request_handlers.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/custom_router_default_only.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/error_handler.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/failed_request_handler.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/http_pre_navigation.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/playwright_pre_navigation.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/router_middleware.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_router/simple_default_handler.py (100%) rename docs/{guides => 03_concepts}/code_examples/request_throttling/throttling_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/scaling_crawlers/max_tasks_per_minute_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/scaling_crawlers/min_and_max_concurrency_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_conflicts.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_crawler_configuration.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_crawler_event_manager.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_crawler_storage_client.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_locator_configuration.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_locator_event_manager.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_locator_storage_client.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_storage_configuration.py (100%) rename docs/{guides => 03_concepts}/code_examples/service_locator/service_storage_storage_client.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/multi_sessions_http.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/one_session_http.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/sm_basic.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/sm_beautifulsoup.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/sm_http.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/sm_parsel.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/sm_playwright.py (100%) rename docs/{guides => 03_concepts}/code_examples/session_management/sm_standalone.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/custom_storage_client_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/file_system_storage_client_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/file_system_storage_client_configuration_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/memory_storage_client_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/redis_storage_client_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/redis_storage_client_configuration_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/registering_storage_clients_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/sql_storage_client_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storage_clients/sql_storage_client_configuration_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/cleaning_do_not_purge_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/cleaning_purge_explicitly_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/dataset_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/dataset_with_crawler_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/dataset_with_crawler_explicit_example.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/storages}/export_entire_dataset_to_file_csv.py (100%) rename docs/{examples/code_examples => 03_concepts/code_examples/storages}/export_entire_dataset_to_file_json.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/helper_add_requests_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/helper_enqueue_links_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/kvs_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/kvs_with_crawler_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/kvs_with_crawler_explicit_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/opening.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/rq_basic_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/rq_with_crawler_example.py (100%) rename docs/{guides => 03_concepts}/code_examples/storages/rq_with_crawler_explicit_example.py (100%) create mode 100644 docs/04_guides/01_crawling_links.mdx rename docs/{examples/fill_and_submit_web_form.mdx => 04_guides/02_fill_and_submit_web_form.mdx} (95%) rename docs/{guides/crawler_login.mdx => 04_guides/03_crawler_login.mdx} (57%) rename docs/{guides/avoid_blocking.mdx => 04_guides/04_avoid_blocking.mdx} (70%) rename docs/{examples/respect_robots_txt_file.mdx => 04_guides/05_respect_robots_txt_file.mdx} (86%) create mode 100644 docs/04_guides/06_stopping_and_resuming.mdx rename docs/{examples/run_parallel_crawlers.mdx => 04_guides/07_run_parallel_crawlers.mdx} (76%) rename docs/{guides/running_in_web_server.mdx => 04_guides/08_running_in_web_server.mdx} (96%) rename docs/{guides/trace_and_monitor_crawlers.mdx => 04_guides/09_trace_and_monitor_crawlers.mdx} (100%) rename docs/{guides/creating_web_archive.mdx => 04_guides/10_creating_web_archive.mdx} (98%) rename docs/{guides/secure_scraping.mdx => 04_guides/11_secure_scraping.mdx} (95%) rename docs/{guides/extending_crawlee.mdx => 04_guides/12_extending_crawlee.mdx} (93%) rename docs/{guides/pydantic_ai_crawler.mdx => 04_guides/13_pydantic_ai_crawler.mdx} (100%) rename docs/{guides/stagehand_crawler.mdx => 04_guides/14_stagehand_crawler.mdx} (97%) rename docs/{guides/scrapy_migration.mdx => 04_guides/15_scrapy_migration.mdx} (92%) rename docs/{guides => 04_guides}/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/avoid_blocking}/playwright_crawler_with_camoufox.py (100%) rename docs/{guides => 04_guides}/code_examples/avoid_blocking/playwright_with_cloakbrowser.py (100%) rename docs/{guides => 04_guides}/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_all_links_on_website_bs.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_all_links_on_website_pw.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_multiple_urls_bs.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_multiple_urls_pw.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_specific_links_on_website_bs.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_specific_links_on_website_pw.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_website_with_relative_links_all_links.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_website_with_relative_links_same_domain.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_website_with_relative_links_same_hostname.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/crawl_website_with_relative_links_same_origin.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/extract_and_add_specific_links_on_website_bs.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/crawling_links}/extract_and_add_specific_links_on_website_pw.py (100%) rename docs/{guides => 04_guides}/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py (100%) rename docs/{guides => 04_guides}/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py (100%) rename docs/{guides => 04_guides}/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/fill_and_submit_web_form}/fill_and_submit_web_form_automated.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/fill_and_submit_web_form}/fill_and_submit_web_form_crawler.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/fill_and_submit_web_form}/fill_and_submit_web_form_request.py (100%) rename docs/{guides => 04_guides}/code_examples/login_crawler/http_login.py (100%) rename docs/{guides => 04_guides}/code_examples/login_crawler/playwright_login.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/login_crawler}/using_browser_profiles_chrome.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/login_crawler}/using_browser_profiles_firefox.py (100%) rename docs/{guides => 04_guides}/code_examples/pydantic_ai_crawler/additional_instructions_example.py (100%) rename docs/{guides => 04_guides}/code_examples/pydantic_ai_crawler/basic_example.py (100%) rename docs/{guides => 04_guides}/code_examples/pydantic_ai_crawler/custom_distiller_example.py (100%) rename docs/{guides => 04_guides}/code_examples/pydantic_ai_crawler/debugging_example.py (100%) rename docs/{guides => 04_guides}/code_examples/pydantic_ai_crawler/selector_extractor_example.py (100%) rename docs/{guides => 04_guides}/code_examples/pydantic_ai_crawler/usage_limit_example.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/respect_robots_txt_file}/respect_robots_on_skipped_request.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/respect_robots_txt_file}/respect_robots_txt_file.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/run_parallel_crawlers}/run_parallel_crawlers.py (100%) rename docs/{introduction/code_examples => 04_guides/code_examples/running_in_web_server}/__init__.py (100%) rename docs/{guides => 04_guides}/code_examples/running_in_web_server/crawler.py (100%) rename docs/{guides => 04_guides}/code_examples/running_in_web_server/server.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_concurrency.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_crawlspider.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_error_handling.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_export.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_labels.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_playwright.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_post.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_proxy.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_quotes.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_throttling.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/crawlee_user_data.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_authors.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_cb_kwargs.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_concurrency.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_crawlspider.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_errback.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_export.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_formrequest.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_playwright.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_playwright_settings.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_proxy.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_quotes.py (100%) rename docs/{guides => 04_guides}/code_examples/scrapy_migration/scrapy_throttling.py (100%) rename docs/{guides => 04_guides}/code_examples/stagehand_crawler/basic_example.py (100%) rename docs/{guides => 04_guides}/code_examples/stagehand_crawler/browserbase_example.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/stopping_and_resuming}/beautifulsoup_crawler_keep_alive.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/stopping_and_resuming}/beautifulsoup_crawler_stop.py (100%) rename docs/{examples/code_examples => 04_guides/code_examples/stopping_and_resuming}/resuming_paused_crawl.py (100%) rename docs/{guides => 04_guides}/code_examples/trace_and_monitor_crawlers/instrument_crawler.py (100%) rename docs/{deployment => 05_deployment}/apify_platform.mdx (97%) rename docs/{deployment => 05_deployment}/aws_lambda.mdx (99%) rename docs/{deployment => 05_deployment}/code_examples/apify/crawler_as_actor_example.py (100%) rename docs/{deployment => 05_deployment}/code_examples/apify/get_public_url.py (100%) rename docs/{deployment => 05_deployment}/code_examples/apify/log_with_config_example.py (100%) rename docs/{deployment => 05_deployment}/code_examples/apify/proxy_advanced_example.py (100%) rename docs/{deployment => 05_deployment}/code_examples/apify/proxy_example.py (100%) rename docs/{deployment => 05_deployment}/code_examples/aws/beautifulsoup_crawler_lambda.py (100%) rename docs/{deployment => 05_deployment}/code_examples/aws/playwright_crawler_lambda.py (100%) rename docs/{deployment => 05_deployment}/code_examples/aws/playwright_dockerfile (100%) rename docs/{deployment => 05_deployment}/code_examples/google/cloud_run_example.py (100%) rename docs/{deployment => 05_deployment}/code_examples/google/google_example.py (100%) rename docs/{deployment => 05_deployment}/google_cloud.mdx (97%) rename docs/{deployment => 05_deployment}/google_cloud_run.mdx (100%) rename docs/{upgrading => 06_upgrading}/upgrading_to_v0x.md (98%) rename docs/{upgrading => 06_upgrading}/upgrading_to_v1.md (97%) delete mode 100644 docs/examples/add_data_to_dataset.mdx delete mode 100644 docs/examples/beautifulsoup_crawler.mdx delete mode 100644 docs/examples/capture_screenshot_using_playwright.mdx delete mode 100644 docs/examples/capturing_page_snapshots_with_error_snapshotter.mdx delete mode 100644 docs/examples/code_examples/add_data_to_dataset_bs.py delete mode 100644 docs/examples/code_examples/add_data_to_dataset_dataset.py delete mode 100644 docs/examples/code_examples/add_data_to_dataset_pw.py delete mode 100644 docs/examples/code_examples/beautifulsoup_crawler.py delete mode 100644 docs/examples/code_examples/parsel_crawler.py delete mode 100644 docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py delete mode 100644 docs/examples/crawl_all_links_on_website.mdx delete mode 100644 docs/examples/crawl_multiple_urls.mdx delete mode 100644 docs/examples/crawl_specific_links_on_website.mdx delete mode 100644 docs/examples/crawl_website_with_relative_links.mdx delete mode 100644 docs/examples/crawler_keep_alive.mdx delete mode 100644 docs/examples/crawler_stop.mdx delete mode 100644 docs/examples/export_entire_dataset_to_file.mdx delete mode 100644 docs/examples/file_download.mdx delete mode 100644 docs/examples/json_logging.mdx delete mode 100644 docs/examples/parsel_crawler.mdx delete mode 100644 docs/examples/playwright_crawler.mdx delete mode 100644 docs/examples/playwright_crawler_adaptive.mdx delete mode 100644 docs/examples/playwright_crawler_with_block_requests.mdx delete mode 100644 docs/examples/playwright_crawler_with_camoufox.mdx delete mode 100644 docs/examples/playwright_crawler_with_fingerprint_generator.mdx delete mode 100644 docs/examples/resuming_paused_crawl.mdx delete mode 100644 docs/examples/using_browser_profile.mdx delete mode 100644 docs/examples/using_sitemap_request_loader.mdx diff --git a/README.md b/README.md index 6fc5b9ef67..6e33b33fab 100644 --- a/README.md +++ b/README.md @@ -167,7 +167,7 @@ if __name__ == '__main__': ### More examples -Explore our [Examples](https://crawlee.dev/python/docs/examples) page in the Crawlee documentation for a wide range of additional use cases and demonstrations. +Explore the [Guides](https://crawlee.dev/python/docs/guides) section of the Crawlee documentation for a wide range of additional use cases and demonstrations. ## Features diff --git a/docs/quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/01_quick-start/code_examples/beautifulsoup_crawler_example.py similarity index 100% rename from docs/quick-start/code_examples/beautifulsoup_crawler_example.py rename to docs/01_quick-start/code_examples/beautifulsoup_crawler_example.py diff --git a/docs/quick-start/code_examples/parsel_crawler_example.py b/docs/01_quick-start/code_examples/parsel_crawler_example.py similarity index 100% rename from docs/quick-start/code_examples/parsel_crawler_example.py rename to docs/01_quick-start/code_examples/parsel_crawler_example.py diff --git a/docs/quick-start/code_examples/playwright_crawler_example.py b/docs/01_quick-start/code_examples/playwright_crawler_example.py similarity index 100% rename from docs/quick-start/code_examples/playwright_crawler_example.py rename to docs/01_quick-start/code_examples/playwright_crawler_example.py diff --git a/docs/quick-start/code_examples/playwright_crawler_headful_example.py b/docs/01_quick-start/code_examples/playwright_crawler_headful_example.py similarity index 100% rename from docs/quick-start/code_examples/playwright_crawler_headful_example.py rename to docs/01_quick-start/code_examples/playwright_crawler_headful_example.py diff --git a/docs/quick-start/index.mdx b/docs/01_quick-start/index.mdx similarity index 92% rename from docs/quick-start/index.mdx rename to docs/01_quick-start/index.mdx index 8125f497d6..c8b61ec0eb 100644 --- a/docs/quick-start/index.mdx +++ b/docs/01_quick-start/index.mdx @@ -1,6 +1,7 @@ --- id: quick-start title: Quick start +description: Build and run your first Crawlee crawler in a few minutes, and pick the crawler type that fits your project. --- import ApiLink from '@site/src/components/ApiLink'; @@ -15,7 +16,7 @@ import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/pl import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/playwright_crawler_headful_example.py'; -This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](../introduction/index.mdx) section, which provides a comprehensive step-by-step guide to creating your first scraper. +This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](../02_introduction/index.mdx) section, which provides a comprehensive step-by-step guide to creating your first scraper. ## Choose your crawler @@ -61,7 +62,7 @@ If you plan to use the `PlaywrightCrawler` playwright install ``` -For detailed installation instructions, see the [Setting up](../introduction/01_setting_up.mdx) documentation page. +For detailed installation instructions, see the [Setting up](../02_introduction/01_setting_up.mdx) documentation page. ## Crawling @@ -128,6 +129,6 @@ If you want to change the storage directory, you can set the `CRAWLEE_STORAGE_DI ## Examples and further reading -For more examples showcasing various features of Crawlee, visit the [Examples](/docs/examples) section of the documentation. To get a deeper understanding of Crawlee and its components, read the step-by-step [Introduction](../introduction/index.mdx) guide. +For more examples showcasing various features of Crawlee, visit the [Guides](./guides) section of the documentation. To get a deeper understanding of Crawlee and its components, read the step-by-step [Introduction](../02_introduction/index.mdx) guide. [//]: # (TODO: add related links once they are ready) diff --git a/docs/introduction/01_setting_up.mdx b/docs/02_introduction/01_setting_up.mdx similarity index 98% rename from docs/introduction/01_setting_up.mdx rename to docs/02_introduction/01_setting_up.mdx index 4c5215a576..75858d517c 100644 --- a/docs/introduction/01_setting_up.mdx +++ b/docs/02_introduction/01_setting_up.mdx @@ -1,6 +1,7 @@ --- id: setting-up title: Setting up +description: How to install Crawlee, set up your environment, and bootstrap a new project. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/introduction/02_first_crawler.mdx b/docs/02_introduction/02_first_crawler.mdx similarity index 98% rename from docs/introduction/02_first_crawler.mdx rename to docs/02_introduction/02_first_crawler.mdx index 0e8d5bd6e0..17e77345e0 100644 --- a/docs/introduction/02_first_crawler.mdx +++ b/docs/02_introduction/02_first_crawler.mdx @@ -1,6 +1,7 @@ --- id: first-crawler title: First crawler +description: Build your first Crawlee crawler - set up a request queue, write a request handler, and crawl your first page. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/introduction/03_adding_more_urls.mdx b/docs/02_introduction/03_adding_more_urls.mdx similarity index 98% rename from docs/introduction/03_adding_more_urls.mdx rename to docs/02_introduction/03_adding_more_urls.mdx index bfe2337d07..b183be04b5 100644 --- a/docs/introduction/03_adding_more_urls.mdx +++ b/docs/02_introduction/03_adding_more_urls.mdx @@ -1,6 +1,7 @@ --- id: adding-more-urls title: Adding more URLs +description: Grow the crawl by discovering and enqueuing new links, with filtering and deduplication handled for you. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/introduction/04_real_world_project.mdx b/docs/02_introduction/04_real_world_project.mdx similarity index 98% rename from docs/introduction/04_real_world_project.mdx rename to docs/02_introduction/04_real_world_project.mdx index 61f6435980..21910a488d 100644 --- a/docs/introduction/04_real_world_project.mdx +++ b/docs/02_introduction/04_real_world_project.mdx @@ -1,6 +1,7 @@ --- id: real-world-project title: Real-world project +description: Plan a real scraping project - choose the data to collect and analyze the target website before writing code. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/introduction/05_crawling.mdx b/docs/02_introduction/05_crawling.mdx similarity index 96% rename from docs/introduction/05_crawling.mdx rename to docs/02_introduction/05_crawling.mdx index 7c68662766..412db44d82 100644 --- a/docs/introduction/05_crawling.mdx +++ b/docs/02_introduction/05_crawling.mdx @@ -1,6 +1,7 @@ --- id: crawling title: Crawling +description: Crawl the example Warehouse store - visit the category listings and enqueue the product detail pages. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/introduction/06_scraping.mdx b/docs/02_introduction/06_scraping.mdx similarity index 98% rename from docs/introduction/06_scraping.mdx rename to docs/02_introduction/06_scraping.mdx index 51c86e5835..cd95249009 100644 --- a/docs/introduction/06_scraping.mdx +++ b/docs/02_introduction/06_scraping.mdx @@ -1,6 +1,7 @@ --- id: scraping title: Scraping +description: Extract structured data such as titles, prices, and stock information from the product detail pages. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/introduction/07_saving_data.mdx b/docs/02_introduction/07_saving_data.mdx similarity index 86% rename from docs/introduction/07_saving_data.mdx rename to docs/02_introduction/07_saving_data.mdx index adddd93af9..7374f89645 100644 --- a/docs/introduction/07_saving_data.mdx +++ b/docs/02_introduction/07_saving_data.mdx @@ -1,6 +1,7 @@ --- id: saving-data title: Saving data +description: Persist the scraped results into a dataset and find them on disk. --- import ApiLink from '@site/src/components/ApiLink'; @@ -88,19 +89,10 @@ A helper `context.push_data` save :::info Automatic dataset initialization -Each time you start Crawlee a default `Dataset` is automatically created, so there's no need to initialize it or create an instance first. You can create as many datasets as you want and even give them names. For more details see the `Dataset.open` function. +Each time you start Crawlee a default `Dataset` is automatically created, so there's no need to initialize it or create an instance first. You can create as many datasets as you want and even give them names. For more details see the [Storages](../concepts/storages#dataset) page and the `Dataset.open` function. ::: -{/* TODO: mention result storage guide once it's done - -:::info Automatic dataset initialization - -Each time you start Crawlee a default `Dataset` is automatically created, so there's no need to initialize it or create an instance first. You can create as many datasets as you want and even give them names. For more details see the [Result storage guide](../guides/result-storage#dataset) and the `Dataset.open()` function. - -::: -*/} - ## Finding saved data Unless you changed the configuration that Crawlee uses locally, which would suggest that you knew what you were doing, and you didn't need this tutorial anyway, you'll find your data in the storage directory that Crawlee creates in the working directory of the running script: @@ -111,16 +103,12 @@ Unless you changed the configuration that Crawlee uses locally, which would sugg The above folder will hold all your saved data in numbered files, as they were pushed into the dataset. Each file represents one invocation of `Dataset.push_data` or one table row. -{/* TODO: add mention of "Result storage guide" once it's ready: - :::tip Single file data storage options -If you would like to store your data in a single big file, instead of many small ones, see the [Result storage guide](../guides/result-storage#key-value-store) for Key-value stores. +If you would like to store your data in a single big file, instead of many small ones, see how to [export the whole dataset](../concepts/storages#exporting-the-dataset) to JSON or CSV, or use a [key-value store](../concepts/storages#key-value-store). ::: -*/} - ## Next steps Next, you'll see some improvements that you can add to your crawler code that will make it more readable and maintainable in the long run. diff --git a/docs/introduction/08_refactoring.mdx b/docs/02_introduction/08_refactoring.mdx similarity index 94% rename from docs/introduction/08_refactoring.mdx rename to docs/02_introduction/08_refactoring.mdx index a194a9e839..7245e576f3 100644 --- a/docs/introduction/08_refactoring.mdx +++ b/docs/02_introduction/08_refactoring.mdx @@ -1,6 +1,7 @@ --- id: refactoring title: Refactoring +description: Clean up the crawler code with a router and separate handlers to keep the project maintainable. --- import ApiLink from '@site/src/components/ApiLink'; @@ -21,7 +22,7 @@ You might be wondering about the **anti-blocking, bot-protection avoiding stealt However, the default configuration, while powerful, may not cover every scenario. -If you want to learn more, browse the [Avoid getting blocked](../guides/avoid-blocking), [Proxy management](../guides/proxy-management) and [Session management](../guides/session-management) guides. +If you want to learn more, browse the [Avoid getting blocked](../guides/avoid-blocking), [Proxy management](../concepts/proxy-management) and [Session management](../concepts/session-management) guides. */} To promote good coding practices, let's look at how you can use a `Router` class to better structure your crawler code. diff --git a/docs/introduction/09_running_in_cloud.mdx b/docs/02_introduction/09_running_in_cloud.mdx similarity index 98% rename from docs/introduction/09_running_in_cloud.mdx rename to docs/02_introduction/09_running_in_cloud.mdx index db8273f94f..c6fb8a5859 100644 --- a/docs/introduction/09_running_in_cloud.mdx +++ b/docs/02_introduction/09_running_in_cloud.mdx @@ -2,7 +2,7 @@ id: deployment title: Running your crawler in the Cloud sidebar_label: Running in the Cloud -description: Deploying Crawlee-python projects to the Apify platform +description: Deploy your Crawlee for Python project to the Apify platform and run it in the cloud. --- import CodeBlock from '@theme/CodeBlock'; diff --git a/docs/introduction/code_examples/02_bs.py b/docs/02_introduction/code_examples/02_bs.py similarity index 100% rename from docs/introduction/code_examples/02_bs.py rename to docs/02_introduction/code_examples/02_bs.py diff --git a/docs/introduction/code_examples/02_bs_better.py b/docs/02_introduction/code_examples/02_bs_better.py similarity index 100% rename from docs/introduction/code_examples/02_bs_better.py rename to docs/02_introduction/code_examples/02_bs_better.py diff --git a/docs/introduction/code_examples/02_request_queue.py b/docs/02_introduction/code_examples/02_request_queue.py similarity index 100% rename from docs/introduction/code_examples/02_request_queue.py rename to docs/02_introduction/code_examples/02_request_queue.py diff --git a/docs/introduction/code_examples/03_enqueue_strategy.py b/docs/02_introduction/code_examples/03_enqueue_strategy.py similarity index 100% rename from docs/introduction/code_examples/03_enqueue_strategy.py rename to docs/02_introduction/code_examples/03_enqueue_strategy.py diff --git a/docs/introduction/code_examples/03_finding_new_links.py b/docs/02_introduction/code_examples/03_finding_new_links.py similarity index 100% rename from docs/introduction/code_examples/03_finding_new_links.py rename to docs/02_introduction/code_examples/03_finding_new_links.py diff --git a/docs/introduction/code_examples/03_globs.py b/docs/02_introduction/code_examples/03_globs.py similarity index 100% rename from docs/introduction/code_examples/03_globs.py rename to docs/02_introduction/code_examples/03_globs.py diff --git a/docs/introduction/code_examples/03_original_code.py b/docs/02_introduction/code_examples/03_original_code.py similarity index 100% rename from docs/introduction/code_examples/03_original_code.py rename to docs/02_introduction/code_examples/03_original_code.py diff --git a/docs/introduction/code_examples/03_transform_request.py b/docs/02_introduction/code_examples/03_transform_request.py similarity index 100% rename from docs/introduction/code_examples/03_transform_request.py rename to docs/02_introduction/code_examples/03_transform_request.py diff --git a/docs/introduction/code_examples/04_sanity_check.py b/docs/02_introduction/code_examples/04_sanity_check.py similarity index 100% rename from docs/introduction/code_examples/04_sanity_check.py rename to docs/02_introduction/code_examples/04_sanity_check.py diff --git a/docs/introduction/code_examples/05_crawling_detail.py b/docs/02_introduction/code_examples/05_crawling_detail.py similarity index 100% rename from docs/introduction/code_examples/05_crawling_detail.py rename to docs/02_introduction/code_examples/05_crawling_detail.py diff --git a/docs/introduction/code_examples/05_crawling_listing.py b/docs/02_introduction/code_examples/05_crawling_listing.py similarity index 100% rename from docs/introduction/code_examples/05_crawling_listing.py rename to docs/02_introduction/code_examples/05_crawling_listing.py diff --git a/docs/introduction/code_examples/06_scraping.py b/docs/02_introduction/code_examples/06_scraping.py similarity index 100% rename from docs/introduction/code_examples/06_scraping.py rename to docs/02_introduction/code_examples/06_scraping.py diff --git a/docs/introduction/code_examples/07_final_code.py b/docs/02_introduction/code_examples/07_final_code.py similarity index 100% rename from docs/introduction/code_examples/07_final_code.py rename to docs/02_introduction/code_examples/07_final_code.py diff --git a/docs/introduction/code_examples/07_first_code.py b/docs/02_introduction/code_examples/07_first_code.py similarity index 100% rename from docs/introduction/code_examples/07_first_code.py rename to docs/02_introduction/code_examples/07_first_code.py diff --git a/docs/introduction/code_examples/08_main.py b/docs/02_introduction/code_examples/08_main.py similarity index 100% rename from docs/introduction/code_examples/08_main.py rename to docs/02_introduction/code_examples/08_main.py diff --git a/docs/introduction/code_examples/08_routes.py b/docs/02_introduction/code_examples/08_routes.py similarity index 100% rename from docs/introduction/code_examples/08_routes.py rename to docs/02_introduction/code_examples/08_routes.py diff --git a/docs/introduction/code_examples/09_apify_sdk.py b/docs/02_introduction/code_examples/09_apify_sdk.py similarity index 100% rename from docs/introduction/code_examples/09_apify_sdk.py rename to docs/02_introduction/code_examples/09_apify_sdk.py diff --git a/docs/guides/code_examples/http_crawlers/__init__.py b/docs/02_introduction/code_examples/__init__.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/__init__.py rename to docs/02_introduction/code_examples/__init__.py diff --git a/docs/introduction/code_examples/routes.py b/docs/02_introduction/code_examples/routes.py similarity index 100% rename from docs/introduction/code_examples/routes.py rename to docs/02_introduction/code_examples/routes.py diff --git a/docs/introduction/index.mdx b/docs/02_introduction/index.mdx similarity index 94% rename from docs/introduction/index.mdx rename to docs/02_introduction/index.mdx index 138c6fddb0..fc7d3dab86 100644 --- a/docs/introduction/index.mdx +++ b/docs/02_introduction/index.mdx @@ -1,6 +1,7 @@ --- id: introduction title: Introduction +description: A step-by-step tutorial that takes you from your first crawler to a production-ready scraper for a real website. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/guides/architecture_overview.mdx b/docs/03_concepts/01_architecture_overview.mdx similarity index 91% rename from docs/guides/architecture_overview.mdx rename to docs/03_concepts/01_architecture_overview.mdx index 672e648d36..21b2b7ef7c 100644 --- a/docs/guides/architecture_overview.mdx +++ b/docs/03_concepts/01_architecture_overview.mdx @@ -6,7 +6,7 @@ description: An overview of the core components of the Crawlee library and its a import ApiLink from '@site/src/components/ApiLink'; -Crawlee is a modern and modular web scraping framework. It is designed for both HTTP-only and browser-based scraping. In this guide, we will provide a high-level overview of its architecture and the main components that make up the system. +Crawlee is a modern and modular web scraping framework. It is designed for both HTTP-only and browser-based scraping. This page provides a high-level overview of its architecture and the main components that make up the system. ## Crawler @@ -76,7 +76,7 @@ PlaywrightCrawler --|> StagehandCrawler ### HTTP crawlers -HTTP crawlers use HTTP clients to fetch pages and parse them with HTML parsing libraries. They are fast and efficient for sites that do not require JavaScript rendering. HTTP clients are Crawlee components that wrap around HTTP libraries like [httpx2](https://httpx2.pydantic.dev/), [curl-impersonate](https://github.com/lwthiker/curl-impersonate) or [impit](https://github.com/apify/impit) and handle HTTP communication for requests and responses. You can learn more about them in the [HTTP clients guide](./http-clients). +HTTP crawlers use HTTP clients to fetch pages and parse them with HTML parsing libraries. They are fast and efficient for sites that do not require JavaScript rendering. HTTP clients are Crawlee components that wrap around HTTP libraries like [httpx2](https://httpx2.pydantic.dev/), [curl-impersonate](https://github.com/lwthiker/curl-impersonate) or [impit](https://github.com/apify/impit) and handle HTTP communication for requests and responses. You can learn more about them on the [HTTP clients](./http-clients) page. HTTP crawlers inherit from `AbstractHttpCrawler` and there are five crawlers that belong to this category: @@ -86,18 +86,18 @@ HTTP crawlers inherit from `AbstractHttp - `PydanticAiCrawler` parses HTML with Parsel and uses an LLM to extract structured data into a validated Pydantic model. - `FileDownloadCrawler` downloads files of any content type, optionally streaming large bodies in chunks. -You can learn more about HTTP crawlers in the [HTTP crawlers guide](./http-crawlers). +You can learn more about HTTP crawlers on the [HTTP crawlers](./http-crawlers) page. ### Browser crawlers Browser crawlers use a real browser to render pages, enabling scraping of sites that require JavaScript. They manage browser instances, pages, and context lifecycles. Crawlee provides two browser crawlers: -- `PlaywrightCrawler` utilizes the [Playwright](https://playwright.dev/) library and provides a high-level API for controlling and navigating browsers. You can learn more about it in the [Playwright crawler guide](./playwright-crawler). -- `StagehandCrawler` extends `PlaywrightCrawler` with AI-powered browser automation via [Stagehand](https://github.com/browserbase/stagehand). It adds natural-language methods (`act`, `extract`, `observe`, `execute`) directly on the page object. You can learn more about it in the [Stagehand crawler guide](./stagehand-crawler). +- `PlaywrightCrawler` utilizes the [Playwright](https://playwright.dev/) library and provides a high-level API for controlling and navigating browsers. You can learn more about it on the [Playwright crawler](./playwright-crawler) page. +- `StagehandCrawler` extends `PlaywrightCrawler` with AI-powered browser automation via [Stagehand](https://github.com/browserbase/stagehand). It adds natural-language methods (`act`, `extract`, `observe`, `execute`) directly on the page object. You can learn more about it in the [Stagehand crawler guide](../guides/stagehand-crawler). ### Adaptive crawler -The `AdaptivePlaywrightCrawler` sits between HTTP and browser crawlers. It can automatically decide whether to use HTTP or browser crawling for each request based on heuristics or user configuration. This allows for optimal performance and compatibility. It also provides a uniform interface for both crawling types (modes). You can learn more about adaptive crawling in the [Adaptive Playwright crawler guide](./adaptive-playwright-crawler). +The `AdaptivePlaywrightCrawler` sits between HTTP and browser crawlers. It can automatically decide whether to use HTTP or browser crawling for each request based on heuristics or user configuration. This allows for optimal performance and compatibility. It also provides a uniform interface for both crawling types (modes). You can learn more about adaptive crawling on the [Adaptive Playwright crawler](./adaptive-playwright-crawler) page. ## Crawling contexts @@ -208,7 +208,7 @@ Crawlee provides three built-in storage types for managing data: - `KeyValueStore` - Storage for arbitrary data like JSON documents, images or configs. It supports get and set operations with key-value pairs; updates are only possible by replacement. - `RequestQueue` - A managed queue for pending and completed requests, with automatic deduplication and dynamic addition of new items. It is used to track URLs for crawling. -See the [Storages guide](./storages) for more details. +See the [Storages](./storages) page for more details. ```mermaid --- @@ -294,7 +294,7 @@ StorageClient --|> ApifyStorageClient Storage clients can be registered globally with the `ServiceLocator` (you will learn more about the `ServiceLocator` in the next section), passed directly to crawlers, or specified when opening individual storage instances. You can also create custom storage clients by implementing the `StorageClient` interface. -See the [Storage clients guide](./storage-clients) for more details. +See the [Storage clients](./storage-clients) page for more details. ## Request router @@ -312,7 +312,7 @@ The request routing in Crawlee supports: - Failed request handlers - Handle requests that exceed retry limits. - Pre-navigation hooks - Execute logic before navigating to URLs. -See the [Request router guide](./request-router) for detailed information and examples. +See the [Request router](./request-router) page for detailed information and examples. ## Service locator @@ -324,7 +324,7 @@ The `ServiceLocator` is a central r Services can be registered globally through the `service_locator` singleton instance, passed to crawler constructors, or provided when opening individual storage instances. The service locator includes conflict prevention mechanisms to ensure configuration consistency and prevent accidental service conflicts during runtime. -See the [Service locator guide](./service-locator) for detailed information about service registration and configuration options. +See the [Service locator](./service-locator) page for detailed information about service registration and configuration options. ## Request loaders @@ -342,7 +342,7 @@ Request loaders provide a subset of `RequestQue Request loaders are useful when you need to start with a predefined set of URLs. The tandem approach allows processing requests from static sources (like files or sitemaps) while maintaining the ability to add new requests dynamically. -See the [Request loaders guide](./request-loaders) for detailed information. +See the [Request loaders](./request-loaders) page for detailed information. ## Event manager @@ -356,7 +356,7 @@ Crawlee provides several implementations of the event manager: :::info -You can learn more about `Snapshotter` and `AutoscaledPool` and their configuration in the [Scaling crawlers guide](./scaling-crawlers). +You can learn more about `Snapshotter` and `AutoscaledPool` and their configuration on the [Scaling crawlers](./scaling-crawlers) page. ::: @@ -412,7 +412,7 @@ The core component of session management in Crawlee is Statistics are logged at configurable intervals in both table and inline formats, with final summary data returned from the `crawler.run` method available through `FinalStatistics`. -## Conclusion - -In this guide, we provided a high-level overview of the core components of the Crawlee library and its architecture. We covered the main components like crawlers, crawling contexts, storages, request routers, service locator, request loaders, event manager, session management, and statistics. Check out other guides, the [API reference](https://crawlee.dev/python/api), and [Examples](../examples) for more details on how to use these components in your own projects. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/http_crawlers.mdx b/docs/03_concepts/02_http_crawlers.mdx similarity index 85% rename from docs/guides/http_crawlers.mdx rename to docs/03_concepts/02_http_crawlers.mdx index 00100d5e46..e2e35437f7 100644 --- a/docs/guides/http_crawlers.mdx +++ b/docs/03_concepts/02_http_crawlers.mdx @@ -20,17 +20,20 @@ import LexborParser from '!!raw-loader!roa-loader!./code_examples/http_crawlers/ import PyqueryParser from '!!raw-loader!roa-loader!./code_examples/http_crawlers/pyquery_parser.py'; import ScraplingParser from '!!raw-loader!roa-loader!./code_examples/http_crawlers/scrapling_parser.py'; +import FileDownloadExample from '!!raw-loader!roa-loader!./code_examples/http_crawlers/file_download.py'; +import FileDownloadStreamExample from '!!raw-loader!./code_examples/http_crawlers/file_download_stream.py'; + import SelectolaxParserSource from '!!raw-loader!./code_examples/http_crawlers/selectolax_parser.py'; import SelectolaxContextSource from '!!raw-loader!./code_examples/http_crawlers/selectolax_context.py'; import SelectolaxCrawlerSource from '!!raw-loader!./code_examples/http_crawlers/selectolax_crawler.py'; import SelectolaxCrawlerRunSource from '!!raw-loader!./code_examples/http_crawlers/selectolax_crawler_run.py'; import AdaptiveCrawlerRunSource from '!!raw-loader!./code_examples/http_crawlers/selectolax_adaptive_run.py'; -HTTP crawlers are ideal for extracting data from server-rendered websites that don't require JavaScript execution. These crawlers make requests via HTTP clients to fetch HTML content and then parse it using various parsing libraries. For client-side rendered content, where you need to execute JavaScript consider using [Playwright crawler](https://crawlee.dev/python/docs/guides/playwright-crawler) instead. +HTTP crawlers are ideal for extracting data from server-rendered websites that don't require JavaScript execution. These crawlers make requests via HTTP clients to fetch HTML content and then parse it using various parsing libraries. For client-side rendered content, where you need to execute JavaScript, consider using the [Playwright crawler](./playwright-crawler) instead. ## Overview -All HTTP crawlers share a common architecture built around the `AbstractHttpCrawler` base class. The main differences lie in the parsing strategy and the context provided to request handlers. There are `BeautifulSoupCrawler`, `ParselCrawler`, `HttpCrawler`, and `FileDownloadCrawler`. It can also be extended to create custom crawlers with specialized parsing requirements. They use HTTP clients to fetch page content and parsing libraries to extract data from the HTML, check out the [HTTP clients guide](./http-clients) to learn about the HTTP clients used by these crawlers, how to switch between them, and how to create custom HTTP clients tailored to your specific requirements. +All HTTP crawlers share a common architecture built around the `AbstractHttpCrawler` base class. The main differences lie in the parsing strategy and the context provided to request handlers. There are `BeautifulSoupCrawler`, `ParselCrawler`, `HttpCrawler`, and `FileDownloadCrawler`. It can also be extended to create custom crawlers with specialized parsing requirements. They use HTTP clients to fetch page content and parsing libraries to extract data from the HTML, check out the [HTTP clients](./http-clients) page to learn about the HTTP clients used by these crawlers, how to switch between them, and how to create custom HTTP clients tailored to your specific requirements. ```mermaid --- @@ -136,7 +139,19 @@ The following examples demonstrate how to integrate with several popular parsing ## FileDownloadCrawler -The `FileDownloadCrawler` downloads files instead of scraping pages. It accepts any content type without parsing and gives the request handler direct access to the response body. By default the whole file is buffered in memory. For large files, construct the crawler with `stream=True` and consume the body in chunks via `read_stream()`. For usage, see the [Download files](../examples/file-download) example. +The `FileDownloadCrawler` downloads files such as PDFs, images or videos instead of scraping pages. It accepts any content type without parsing and gives the request handler direct access to the response body. Each downloaded file can be saved to the `KeyValueStore` together with the content type reported by the server. + + + {FileDownloadExample} + + +### Streaming large files + +By default the whole file is buffered in memory, which doesn't scale to large downloads. Construct the crawler with `stream=True` and the request handler receives a response whose body hasn't been read yet. Consume it in chunks with `read_stream()` and write each chunk to disk as it arrives. + + + {FileDownloadStreamExample} + ## Custom HTTP crawler @@ -192,9 +207,3 @@ The custom crawler works like any built-in crawler. Request handlers receive you - -## Conclusion - -This guide provided a comprehensive overview of HTTP crawlers in Crawlee. You learned about the three main crawler types - `BeautifulSoupCrawler` for fault-tolerant HTML parsing, `ParselCrawler` for high-performance extraction with XPath and CSS selectors, and `HttpCrawler` for raw response processing. You also discovered how to integrate third-party parsing libraries with `HttpCrawler` and how to create fully custom crawlers using `AbstractHttpCrawler` for specialized parsing requirements. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/playwright_crawler.mdx b/docs/03_concepts/03_playwright_crawler.mdx similarity index 75% rename from docs/guides/playwright_crawler.mdx rename to docs/03_concepts/03_playwright_crawler.mdx index 26a03c79e2..c291de3be4 100644 --- a/docs/guides/playwright_crawler.mdx +++ b/docs/03_concepts/03_playwright_crawler.mdx @@ -8,12 +8,15 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; +import BasicExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/playwright_crawler.py'; import MultipleLaunchExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/multiple_launch_example.py'; import BrowserConfigurationExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/browser_configuration_example.py'; import NavigationHooksExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/navigation_hooks_example.py'; import BrowserPoolLaunchHooksExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/browser_pool_launch_hooks_example.py'; import BrowserPoolPageHooksExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/browser_pool_page_hooks_example.py'; import PluginBrowserConfigExample from '!!raw-loader!./code_examples/playwright_crawler/plugin_browser_configuration_example.py'; +import BlockRequestsExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/playwright_block_requests.py'; +import CaptureScreenshotExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler/capture_screenshot_using_playwright.py'; A `PlaywrightCrawler` is a browser-based crawler. In contrast to HTTP-based crawlers like `ParselCrawler` or `BeautifulSoupCrawler`, it uses a real browser to render pages and extract data. It is built on top of the [Playwright](https://playwright.dev/python/) browser automation library. While browser-based crawlers are typically slower and less efficient than HTTP-based crawlers, they can handle dynamic, client-side rendered sites that standard HTTP-based crawlers cannot manage. @@ -25,7 +28,15 @@ Use `PlaywrightCrawler` in scena - **Anti-scraping protection**: Helpful for sites using JavaScript-based security or advanced anti-automation measures. - **Complex cookie management**: Necessary for sites with session or cookie requirements that standard HTTP-based crawlers cannot handle easily. -If [HTTP-based crawlers](https://crawlee.dev/python/docs/guides/http-crawlers) are insufficient, `PlaywrightCrawler` can address these challenges. See a [basic example](../examples/playwright-crawler) for a typical usage demonstration. +If [HTTP-based crawlers](./http-crawlers) are insufficient, `PlaywrightCrawler` can address these challenges. + +## Basic example + +The following example crawls the Hacker News website using headless Chromium. The crawler manages the browser and page instances for you. In the request handler, Playwright's API is used to extract the title, rank, and URL of each post, and links to the next pages are enqueued to keep the crawl going. + + + {BasicExample} + ## Advanced configuration @@ -64,7 +75,7 @@ Create a subclass only when an integration uses a different browser launch API. Keep the inherited plugin configuration intact so that browser context options remain available to `BrowserPool`. When constructing the controller, explicitly preserve or intentionally replace `use_incognito_pages`, `max_open_pages_per_browser`, and fingerprint or header generation behavior. The custom launch path must likewise define how it applies browser launch options and `user_data_dir`. -See the [Camoufox example](../examples/playwright-crawler-with-camoufox) for a complete implementation that uses a different browser launcher. +See the [Camoufox example](../guides/avoid-blocking#using-camoufox) for a complete implementation that uses a different browser launcher. ## Browser pool lifecycle hooks @@ -88,12 +99,30 @@ For additional setup or event-driven actions around page creation and closure, t ## Navigation hooks -Navigation hooks allow for additional configuration at specific points during page navigation. The `pre_navigation_hook` is called before each navigation and provides `PlaywrightPreNavCrawlingContext` - including the [page](https://playwright.dev/python/docs/api/class-page) instance and a `block_requests` helper for filtering unwanted resource types and URL patterns. See the [block requests example](https://crawlee.dev/python/docs/examples/playwright-crawler-with-block-requests) for a dedicated walkthrough. Similarly, the `post_navigation_hook` is called after each navigation and provides `PlaywrightPostNavCrawlingContext` - useful for post-load checks such as detecting CAPTCHAs or verifying page state. +Navigation hooks allow for additional configuration at specific points during page navigation. The `pre_navigation_hook` is called before each navigation and provides `PlaywrightPreNavCrawlingContext` - including the [page](https://playwright.dev/python/docs/api/class-page) instance and a `block_requests` helper for filtering unwanted resource types and URL patterns, covered in [Blocking network requests](#blocking-network-requests). Similarly, the `post_navigation_hook` is called after each navigation and provides `PlaywrightPostNavCrawlingContext` - useful for post-load checks such as detecting CAPTCHAs or verifying page state. {NavigationHooksExample} -## Conclusion +## Blocking network requests + +Loading non-essential resources like images, styles, or analytics scripts wastes bandwidth and slows down crawling. The `block_requests` helper provides the most efficient way to block such requests, as it operates directly in the browser. By default, it blocks URLs matching the following patterns: + +```python +['.css', '.webp', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'] +``` + +You can replace the default patterns with your own by providing `url_patterns`, or extend them by passing additional patterns in `extra_url_patterns`. -This guide introduced the `PlaywrightCrawler` and explained how to configure it using `BrowserPool` and `PlaywrightBrowserPlugin`. You learned how to launch multiple browsers, configure browser and context settings, use `BrowserPool` lifecycle hooks, and apply navigation hooks. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! + + {BlockRequestsExample} + + +## Capturing screenshots + +Playwright's `page.screenshot()` method makes it easy to capture screenshots of the pages your crawler visits. The following example stores each screenshot in the key-value store as a PNG image, under a unique key generated from the page URL. + + + {CaptureScreenshotExample} + diff --git a/docs/guides/playwright_crawler_adaptive.mdx b/docs/03_concepts/04_playwright_crawler_adaptive.mdx similarity index 95% rename from docs/guides/playwright_crawler_adaptive.mdx rename to docs/03_concepts/04_playwright_crawler_adaptive.mdx index 7957b98015..da2cca2caf 100644 --- a/docs/guides/playwright_crawler_adaptive.mdx +++ b/docs/03_concepts/04_playwright_crawler_adaptive.mdx @@ -10,6 +10,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; +import AdaptivePlaywrightCrawlerBasicExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py'; import AdaptivePlaywrightCrawlerHandler from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/handler.py'; import AdaptivePlaywrightCrawlerPreNavHooks from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_adaptive/pre_nav_hooks.py'; @@ -28,6 +29,14 @@ Use `AdaptivePlaywrightCrawler` + {AdaptivePlaywrightCrawlerBasicExample} + + ## Request handler and adaptive context helpers Request handler for `AdaptivePlaywrightCrawler` works on special context type - `AdaptivePlaywrightCrawlingContext`. This context is sometimes created by HTTP-based sub crawler and sometimes by playwright based sub crawler. Due to its dynamic nature, you can't always access [page](https://playwright.dev/python/docs/api/class-page) object. To overcome this limitation, there are three helper methods on this context that can be called regardless of how the context was created. diff --git a/docs/guides/request_router.mdx b/docs/03_concepts/05_request_router.mdx similarity index 89% rename from docs/guides/request_router.mdx rename to docs/03_concepts/05_request_router.mdx index 1aceab2637..204a04da25 100644 --- a/docs/guides/request_router.mdx +++ b/docs/03_concepts/05_request_router.mdx @@ -26,7 +26,7 @@ Request handlers are user-defined functions that process individual requests and :::note -The code examples in this guide use `ParselCrawler` for demonstration, but the `Router` works with all crawler types. +The code examples on this page use `ParselCrawler` for demonstration, but the `Router` works with all crawler types. ::: @@ -114,8 +114,3 @@ The `AdaptivePlaywrightCrawler` -## Conclusion - -This guide introduced you to the `Router` class and how to organize your crawling logic. You learned how to use built-in and custom routers, implement request handlers with label-based routing, add middleware with `router.use()`, handle errors with error and failed request handlers, and configure pre-navigation hooks for different crawler types. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/request_loaders.mdx b/docs/03_concepts/06_request_loaders.mdx similarity index 92% rename from docs/guides/request_loaders.mdx rename to docs/03_concepts/06_request_loaders.mdx index a63393142d..8e08bd9035 100644 --- a/docs/guides/request_loaders.mdx +++ b/docs/03_concepts/06_request_loaders.mdx @@ -17,8 +17,9 @@ import SitemapTandemExample from '!!raw-loader!roa-loader!./code_examples/reques import SitemapExplicitTandemExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/sitemap_tandem_example_explicit.py'; import RlBasicPersistExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/rl_basic_example_with_persist.py'; import SitemapPersistExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/sitemap_example_with_persist.py'; +import SitemapTransformExample from '!!raw-loader!roa-loader!./code_examples/request_loaders/using_sitemap_request_loader.py'; -The [`request_loaders`](https://github.com/apify/crawlee-python/tree/master/src/crawlee/request_loaders) sub-package extends the functionality of the `RequestQueue`, providing additional tools for managing URLs and requests. If you are new to Crawlee and unfamiliar with the `RequestQueue`, consider starting with the [Storages](https://crawlee.dev/python/docs/guides/storages) guide first. Request loaders define how requests are fetched and stored, enabling various use cases such as reading URLs from files, external APIs, or combining multiple sources together. +The [`request_loaders`](https://github.com/apify/crawlee-python/tree/master/src/crawlee/request_loaders) sub-package extends the functionality of the `RequestQueue`, providing additional tools for managing URLs and requests. If you are new to Crawlee and unfamiliar with the `RequestQueue`, consider starting with the [Storages](./storages) page first. Request loaders define how requests are fetched and stored, enabling various use cases such as reading URLs from files, external APIs, or combining multiple sources together. ## Overview @@ -122,7 +123,7 @@ Here is a basic example of working with the `Req The `RequestList` supports state persistence, allowing it to resume from where it left off after interruption. This is particularly useful for long-running crawls or when you need to pause and resume crawling later. -To enable persistence, provide `persist_state_key` and optionally `persist_requests_key` parameters, and disable automatic cleanup by setting `purge_on_start = False` in the configuration. The `persist_state_key` saves the loader's progress, while `persist_requests_key` ensures that the request data doesn't change between runs. For more details on resuming interrupted crawls, see the [Resuming a paused crawl](../examples/resuming-paused-crawl) example. +To enable persistence, provide `persist_state_key` and optionally `persist_requests_key` parameters, and disable automatic cleanup by setting `purge_on_start = False` in the configuration. The `persist_state_key` saves the loader's progress, while `persist_requests_key` ensures that the request data doesn't change between runs. For more details on resuming interrupted crawls, see the [Stopping and resuming crawlers](../guides/stopping-and-resuming-crawlers) guide. {RlBasicPersistExample} @@ -152,6 +153,14 @@ Similarly, the `SitemapRequestLoader` + {SitemapTransformExample} + + ## Request managers The `RequestManager` extends `RequestLoader` with write capabilities. In addition to reading requests, a request manager can add and reclaim them. This is essential for dynamic crawling projects where new URLs may emerge during the crawl process, or when certain requests fail and need to be retried. For more details, refer to the `RequestManager` API reference. @@ -196,8 +205,3 @@ Similar to the `RequestList` example a -## Conclusion - -This guide explained the `request_loaders` sub-package, which extends the functionality of the `RequestQueue` with additional tools for managing URLs and requests. You learned about the `RequestLoader`, `RequestManager`, and `RequestManagerTandem` classes, as well as the `RequestList` and `SitemapRequestLoader` implementations. You also saw practical examples of how to work with these classes to handle various crawling scenarios. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/storages.mdx b/docs/03_concepts/07_storages.mdx similarity index 84% rename from docs/guides/storages.mdx rename to docs/03_concepts/07_storages.mdx index a4348e28ab..9157d3cb4f 100644 --- a/docs/guides/storages.mdx +++ b/docs/03_concepts/07_storages.mdx @@ -21,6 +21,9 @@ import DatasetBasicExample from '!!raw-loader!roa-loader!./code_examples/storage import DatasetWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_with_crawler_example.py'; import DatasetWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/dataset_with_crawler_explicit_example.py'; +import ExportJsonExample from '!!raw-loader!roa-loader!./code_examples/storages/export_entire_dataset_to_file_json.py'; +import ExportCsvExample from '!!raw-loader!roa-loader!./code_examples/storages/export_entire_dataset_to_file_csv.py'; + import KvsBasicExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_basic_example.py'; import KvsWithCrawlerExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_with_crawler_example.py'; import KvsWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_examples/storages/kvs_with_crawler_explicit_example.py'; @@ -28,7 +31,7 @@ import KvsWithCrawlerExplicitExample from '!!raw-loader!roa-loader!./code_exampl import CleaningDoNotPurgeExample from '!!raw-loader!roa-loader!./code_examples/storages/cleaning_do_not_purge_example.py'; import CleaningPurgeExplicitlyExample from '!!raw-loader!roa-loader!./code_examples/storages/cleaning_purge_explicitly_example.py'; -Crawlee offers several storage types for managing and persisting your crawling data. Request-oriented storages, such as the `RequestQueue`, help you store and deduplicate URLs, while result-oriented storages, like `Dataset` and `KeyValueStore`, focus on storing and retrieving scraping results. This guide explains when to use each type, how to interact with them, and how to control their lifecycle. +Crawlee offers several storage types for managing and persisting your crawling data. Request-oriented storages, such as the `RequestQueue`, help you store and deduplicate URLs, while result-oriented storages, like `Dataset` and `KeyValueStore`, focus on storing and retrieving scraping results. This page explains when to use each type, how to interact with them, and how to control their lifecycle. ## Overview @@ -37,7 +40,7 @@ Crawlee's storage system consists of two main layers: - **Storages** (`Dataset`, `KeyValueStore`, `RequestQueue`): High-level interfaces for interacting with different storage types. - **Storage clients** (`MemoryStorageClient`, `FileSystemStorageClient`, etc.): Backend implementations that handle the actual data persistence and management. -For more information about storage clients and their configuration, see the [Storage clients guide](./storage-clients). +For more information about storage clients and their configuration, see the [Storage clients](./storage-clients) page. ```mermaid --- @@ -119,7 +122,7 @@ The following code demonstrates the usage of the `RequestQueue`: - The `add_requests` function allows you to manually add specific URLs to the configured request storage. In this case, you must explicitly provide the URLs you want to be added to the request storage. If you need to specify further details of the request, such as a `label` or `user_data`, you have to pass instances of the `Request` class to the helper. -- The `enqueue_links` function is designed to discover new URLs in the current page and add them to the request storage. It can be used with default settings, requiring no arguments, or you can customize its behavior by specifying link element selectors, choosing different enqueue strategies, or applying include/exclude filters to control which URLs are added. See [Crawl website with relative links](../examples/crawl-website-with-relative-links) example for more details. +- The `enqueue_links` function is designed to discover new URLs in the current page and add them to the request storage. It can be used with default settings, requiring no arguments, or you can customize its behavior by specifying link element selectors, choosing different enqueue strategies, or applying include/exclude filters to control which URLs are added. See the [Enqueue strategies](../guides/crawling-links#enqueue-strategies) section of the Crawling links guide for more details. @@ -140,7 +143,7 @@ The `RequestQueue` implements the `RequestManager` class and implementing its required methods. -For a detailed explanation of the `RequestManager` and other related components, refer to the [Request loaders guide](https://crawlee.dev/python/docs/guides/request-loaders). +For a detailed explanation of the `RequestManager` and other related components, refer to the [Request loaders](./request-loaders) page. ## Dataset @@ -172,6 +175,31 @@ Crawlee provides the following helper function to simplify interactions with the - The `push_data` function allows you to manually add data to the dataset. You can optionally specify the dataset ID or its name. +### Exporting the dataset + +The `BasicCrawler.export_data` method exports the entire default dataset to a single file, in either CSV or JSON format. It also accepts additional keyword arguments so you can fine-tune the underlying `json.dump` or `csv.DictWriter` behavior. The method is available on all crawlers. + + + + + {ExportJsonExample} + + + + + {ExportCsvExample} + + + + +Dataset items don't have to share a schema. Different handlers can push different fields, so the items of one dataset often have different keys. By default, the CSV columns are the keys of the first non-empty item. When a later item has a key the first one doesn't, its value isn't written, and Crawlee logs a warning naming the dropped keys. To use the keys of all items as columns instead, pass `collect_all_keys=True`: + +```python +await crawler.export_data(path='results.csv', collect_all_keys=True) +``` + +No value is dropped in that mode. Collecting all keys means reading the whole dataset before the first row can be written, so only the default mode writes rows as it goes. In both modes, cells for columns an item doesn't have stay empty, or hold the `restval` value if you pass one. + ## Key-value store The `KeyValueStore` is designed to save and retrieve data records or files efficiently. Each record is uniquely identified by a key and is associated with a specific MIME type, making the `KeyValueStore` ideal for tasks like saving web page screenshots, PDFs, or tracking the state of crawlers. @@ -196,7 +224,7 @@ The following code demonstrates the usage of the `RequestQueue` and store and retrieve scraping results using the `Dataset` and `KeyValueStore`. You also learned how to use helper functions to simplify interactions with these storages and how to control storage cleanup behavior. +Note that purging behavior may vary between storage client implementations. For more details on storage configuration and client implementations, see the [Storage clients](./storage-clients) page. -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/storage_clients.mdx b/docs/03_concepts/08_storage_clients.mdx similarity index 96% rename from docs/guides/storage_clients.mdx rename to docs/03_concepts/08_storage_clients.mdx index 3186f906c9..ee722b987f 100644 --- a/docs/guides/storage_clients.mdx +++ b/docs/03_concepts/08_storage_clients.mdx @@ -551,8 +551,3 @@ Storage clients can be registered in multiple ways: You can also register different storage clients for each storage instance, allowing you to use different backends for different storages. This is useful when you want to use a fast in-memory storage for `RequestQueue` while persisting scraping results in `Dataset` or `KeyValueStore`. -## Conclusion - -Storage clients in Crawlee provide different backends for data storage. Use `MemoryStorageClient` for testing and fast operations without persistence, or `FileSystemStorageClient` for environments where data needs to persist. You can also create custom storage clients for specialized backends by implementing the `StorageClient` interface. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/http_clients.mdx b/docs/03_concepts/09_http_clients.mdx similarity index 89% rename from docs/guides/http_clients.mdx rename to docs/03_concepts/09_http_clients.mdx index f79fe1941d..04baecef30 100644 --- a/docs/guides/http_clients.mdx +++ b/docs/03_concepts/09_http_clients.mdx @@ -53,7 +53,7 @@ HttpClient --|> CurlImpersonateHttpClient ## Switching between HTTP clients -Crawlee currently provides three main HTTP clients: `ImpitHttpClient`, which uses the `impit` library, `HttpxHttpClient`, which uses the `httpx2` library with `browserforge` for custom HTTP headers and fingerprints, and `CurlImpersonateHttpClient`, which uses the `curl-cffi` library. You can switch between them by setting the `http_client` parameter when initializing a crawler class. The default HTTP client is `ImpitHttpClient`. For more details on anti-blocking features, see our [avoid getting blocked guide](./avoid-blocking). +Crawlee currently provides three main HTTP clients: `ImpitHttpClient`, which uses the `impit` library, `HttpxHttpClient`, which uses the `httpx2` library with `browserforge` for custom HTTP headers and fingerprints, and `CurlImpersonateHttpClient`, which uses the `curl-cffi` library. You can switch between them by setting the `http_client` parameter when initializing a crawler class. The default HTTP client is `ImpitHttpClient`. For more details on anti-blocking features, see our [avoid getting blocked guide](../guides/avoid-blocking). Below are examples of how to configure the HTTP client for the `ParselCrawler`: @@ -111,8 +111,3 @@ HTTP clients are responsible for several key operations: To create a custom HTTP client, you need to inherit from the `HttpClient` base class and implement all required abstract methods. Your implementation must be async-compatible and include proper cleanup and resource management to work seamlessly with Crawlee's concurrent processing model. -## Conclusion - -This guide introduced you to the HTTP clients available in Crawlee and demonstrated how to switch between them, including their installation requirements and usage examples. You also learned about the responsibilities of HTTP clients and how to implement your own custom HTTP client by inheriting from the `HttpClient` base class. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/session_management.mdx b/docs/03_concepts/10_session_management.mdx similarity index 100% rename from docs/guides/session_management.mdx rename to docs/03_concepts/10_session_management.mdx diff --git a/docs/guides/cookie_management.mdx b/docs/03_concepts/11_cookie_management.mdx similarity index 96% rename from docs/guides/cookie_management.mdx rename to docs/03_concepts/11_cookie_management.mdx index c730e7ac0d..cfd93421f0 100644 --- a/docs/guides/cookie_management.mdx +++ b/docs/03_concepts/11_cookie_management.mdx @@ -21,7 +21,7 @@ In Crawlee, cookies are stored on the `Session``PlaywrightCrawler`, and keep them across retries and runs. +This page covers how to read and set cookies, seed them on new sessions, handle them with `PlaywrightCrawler`, and keep them across retries and runs. ## Reading and setting cookies @@ -103,4 +103,4 @@ To see restoration in action, define the pool outside the crawler and read a ses {PersistCookies} -With persistence enabled, a login you performed on a previous run can carry over, so you skip re-authenticating on every start. For a full walkthrough of logging in, see the [Logging in with a crawler](./logging-in-with-a-crawler) guide. +With persistence enabled, a login you performed on a previous run can carry over, so you skip re-authenticating on every start. For a full walkthrough of logging in, see the [Logging in with a crawler](../guides/logging-in-with-a-crawler) guide. diff --git a/docs/guides/http_headers.mdx b/docs/03_concepts/12_http_headers.mdx similarity index 91% rename from docs/guides/http_headers.mdx rename to docs/03_concepts/12_http_headers.mdx index c7c148fe24..b7a23ea8a1 100644 --- a/docs/guides/http_headers.mdx +++ b/docs/03_concepts/12_http_headers.mdx @@ -10,7 +10,7 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import SetHeadersExample from '!!raw-loader!roa-loader!./code_examples/http_headers/set_headers.py'; import BrowserPageHeadersExample from '!!raw-loader!roa-loader!./code_examples/http_headers/browser_page_headers.py'; -Every request a crawler sends includes [HTTP headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers). These headers tell the server who is making the request, what content is acceptable, and in what language. The server reads them and decides what to return. The same URL can return different content, a different status code, or a blocked page depending on the headers it sees. This guide covers the headers that shape a scraping request, like `User-Agent`, `Accept-Language`, and `Content-Type`, what Crawlee sends by default, and how to change them. +Every request a crawler sends includes [HTTP headers](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers). These headers tell the server who is making the request, what content is acceptable, and in what language. The server reads them and decides what to return. The same URL can return different content, a different status code, or a blocked page depending on the headers it sees. This page covers the headers that shape a scraping request, like `User-Agent`, `Accept-Language`, and `Content-Type`, what Crawlee sends by default, and how to change them. ## What headers do @@ -44,7 +44,7 @@ These headers tell the server what the client can handle and the server uses the `Cookie` carries session and login state. Crawlee manages cookies through [sessions](./session-management), so you rarely set this one by hand. For details, see [Cookie management](./cookie-management). -`Authorization` carries credentials, such as a bearer token or basic auth. APIs commonly require it. Set it on the request when the target needs authenticated access. Treat its value as a secret, and don't send it through a [proxy you don't control](./security-of-web-scraping#untrusted-proxies). +`Authorization` carries credentials, such as a bearer token or basic auth. APIs commonly require it. Set it on the request when the target needs authenticated access. Treat its value as a secret, and don't send it through a [proxy you don't control](../guides/security-of-web-scraping#untrusted-proxies). ### Client hints and fingerprinting headers @@ -68,7 +68,7 @@ Each client implements impersonation its own way: - `HttpxHttpClient` uses a `HeaderGenerator` to add `Accept`, `Accept-Language`, and `User-Agent`. - `CurlImpersonateHttpClient` impersonates Chrome at the TLS and HTTP layer through [`curl-cffi`](https://curl-cffi.readthedocs.io/). -The header values match a specific version of a real browser, so the whole set stays internally consistent rather than a mix that no real client would send. For more on staying unblocked, see the [avoid blocking](./avoid-blocking) guide. +The header values match a specific version of a real browser, so the whole set stays internally consistent rather than a mix that no real client would send. For more on staying unblocked, see the [avoid blocking](../guides/avoid-blocking) guide. ## When impersonation hurts @@ -131,8 +131,3 @@ Anti-bot systems look at more than header values. They look at which headers are This fingerprinting is why `ImpitHttpClient` and `CurlImpersonateHttpClient` replicate the browser at the transport layer rather than just attaching headers. Setting a browser `User-Agent` on a plain client isn't enough to pass these checks. If a target uses fingerprinting, prefer an impersonating client over hand-set headers. -## Conclusion - -Headers decide what a server sends back. Crawlee impersonates a browser by default, which keeps a crawl unblocked on normal pages but can break endpoints that expect different headers. Turn impersonation off by building the client without it when you target such an endpoint, set custom headers on the client or per request, and reach for an impersonating client when the target fingerprints its traffic. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/proxy_management.mdx b/docs/03_concepts/13_proxy_management.mdx similarity index 98% rename from docs/guides/proxy_management.mdx rename to docs/03_concepts/13_proxy_management.mdx index 38385ac950..7b6d74a627 100644 --- a/docs/guides/proxy_management.mdx +++ b/docs/03_concepts/13_proxy_management.mdx @@ -1,7 +1,7 @@ --- id: proxy-management title: Proxy management -description: Using proxies to get around those annoying IP-blocks +description: How to use and rotate proxies in Crawlee - proxy configuration, IP rotation tied to sessions, and tiered proxies. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/guides/scaling_crawlers.mdx b/docs/03_concepts/14_scaling_crawlers.mdx similarity index 95% rename from docs/guides/scaling_crawlers.mdx rename to docs/03_concepts/14_scaling_crawlers.mdx index 71dc399107..cad2e5fef4 100644 --- a/docs/guides/scaling_crawlers.mdx +++ b/docs/03_concepts/14_scaling_crawlers.mdx @@ -14,7 +14,7 @@ As we build our crawler, we may want to control how many tasks it performs at an :::tip -All of these options are available across all crawlers provided by Crawlee. In this guide, we are using the `BeautifulSoupCrawler` as an example. You should also explore the `ConcurrencySettings`. +All of these options are available across all crawlers provided by Crawlee. The examples on this page use the `BeautifulSoupCrawler`. You should also explore the `ConcurrencySettings`. ::: diff --git a/docs/guides/request_throttling.mdx b/docs/03_concepts/15_request_throttling.mdx similarity index 100% rename from docs/guides/request_throttling.mdx rename to docs/03_concepts/15_request_throttling.mdx diff --git a/docs/guides/error_handling.mdx b/docs/03_concepts/16_error_handling.mdx similarity index 56% rename from docs/guides/error_handling.mdx rename to docs/03_concepts/16_error_handling.mdx index abd1b33058..99d480d0ea 100644 --- a/docs/guides/error_handling.mdx +++ b/docs/03_concepts/16_error_handling.mdx @@ -5,13 +5,17 @@ description: How to handle errors that occur during web crawling. --- import ApiLink from '@site/src/components/ApiLink'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import HandleProxyError from '!!raw-loader!roa-loader!./code_examples/error_handling/handle_proxy_error.py'; import ChangeHandleErrorStatus from '!!raw-loader!roa-loader!./code_examples/error_handling/change_handle_error_status.py'; import DisableRetry from '!!raw-loader!roa-loader!./code_examples/error_handling/disable_retry.py'; +import ParselCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/error_handling/parsel_crawler_with_error_snapshotter.py'; +import PlaywrightCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/error_handling/playwright_crawler_with_error_snapshotter.py'; -This guide demonstrates techniques for handling common errors encountered during web crawling operations. +This page demonstrates how Crawlee deals with errors during crawling and the techniques available for handling them. ## Handling proxy errors @@ -21,11 +25,11 @@ Low-quality proxies can cause problems even with high settings for `max_request_ {HandleProxyError} -You can use this same approach when testing different proxy providers. To better manage this process, you can count proxy errors and [stop the crawler](../examples/crawler-stop) if you get too many. +You can use this same approach when testing different proxy providers. To better manage this process, you can count proxy errors and [stop the crawler](../guides/stopping-and-resuming-crawlers#stopping-a-crawler) if you get too many. ## Changing how error status codes are handled -By default, when `Sessions` get status codes like [401](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/401), [403](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/403), or [429](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/429), Crawlee marks the `Session` as `retire` and switches to a new one. This might not be what you want, especially when working with [authentication](./logging-in-with-a-crawler). You can learn more in the [Session management guide](./session-management). +By default, when `Sessions` get status codes like [401](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/401), [403](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/403), or [429](https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Status/429), Crawlee marks the `Session` as `retire` and switches to a new one. This might not be what you want, especially when working with [authentication](../guides/logging-in-with-a-crawler). You can learn more on the [Session management](./session-management) page. Here's an example of how to change this behavior: @@ -42,3 +46,20 @@ Here's how to turn off retries for non-network errors using {DisableRetry} + +## Capturing page snapshots on errors + +Debugging a failing request is much easier when you can see what the page looked like at the moment of failure. Set `save_error_snapshots=True` in the crawler's `Statistics` and Crawlee automatically captures a page snapshot on the first occurrence of each unique error. The snapshot can contain an HTML file and a JPEG screenshot of the page where the unhandled exception was raised, and it's saved to the default key-value store. Both `PlaywrightCrawler` and [HTTP crawlers](./http-crawlers) capture the HTML file, but only `PlaywrightCrawler` can capture a screenshot as well. + + + + + {ParselCrawlerWithErrorSnapshotter} + + + + + {PlaywrightCrawlerWithErrorSnapshotter} + + + diff --git a/docs/03_concepts/17_logging.mdx b/docs/03_concepts/17_logging.mdx new file mode 100644 index 0000000000..22fae255ea --- /dev/null +++ b/docs/03_concepts/17_logging.mdx @@ -0,0 +1,70 @@ +--- +id: logging +title: Logging +description: How Crawlee logs its activity and how to configure the log level, statistics logs, and structured JSON output. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import JsonLoggingExample from '!!raw-loader!roa-loader!./code_examples/logging/configure_json_logging.py'; + +Crawlee builds on the standard Python [`logging`](https://docs.python.org/3/library/logging.html) module. Each crawler comes with a logger. HTTP crawlers name it after the crawler class (for example `HttpCrawler`). Inside a request handler the same logger is available as `context.log`, so you can log from your handlers without any setup. + +## Default configuration + +By default, a crawler sets up the logging infrastructure automatically when it's created. It attaches a handler with a human-readable, colored formatter to the root logger, unless the root logger already has handlers configured. You can opt out by passing `configure_logging=False` to the crawler's constructor (see `BasicCrawlerOptions`) and configure logging yourself. + +The log level is controlled by the `Configuration.log_level` option, which can also be set through the `CRAWLEE_LOG_LEVEL` environment variable. + +## Statistics logs + +While running, the crawler periodically logs its statistics - requests finished and failed, retry histogram, durations, and more - and prints a final summary when the crawl ends. The `statistics_log_format` option of the crawler's constructor controls the output format: `table` (the default) renders the statistics as a formatted table, while `inline` logs them as a plain message with the values attached as extra fields, which is easier to consume with external log tooling. + +## JSON logging with loguru + +For log collection and analysis platforms such as the ELK Stack or Grafana Loki, line-delimited JSON (JSONL) output works better than formatted tables. The following example integrates Crawlee with the popular [`loguru`](https://github.com/delgan/loguru) library: an interceptor forwards all standard `logging` records to loguru, which serializes them as one JSON object per line. Note the crawler is created with `configure_logging=False` and `statistics_log_format='inline'`, so the statistics arrive as structured fields instead of a table. + + + {JsonLoggingExample} + + +Here's an example of what a crawler statistics log entry looks like in JSONL format: + +```json +{ + "text": "[HttpCrawler] | INFO | - Final request statistics: {'requests_finished': 1, 'requests_failed': 0, 'retry_histogram': [1], 'request_avg_failed_duration': None, 'request_avg_finished_duration': 3.57098, 'requests_finished_per_minute': 17, 'requests_failed_per_minute': 0, 'request_total_duration': 3.57098, 'requests_total': 1, 'crawler_runtime': 3.59165}\n", + "record": { + "elapsed": { "repr": "0:00:05.604568", "seconds": 5.604568 }, + "exception": null, + "extra": { + "requests_finished": 1, + "requests_failed": 0, + "retry_histogram": [1], + "request_avg_failed_duration": null, + "request_avg_finished_duration": 3.57098, + "requests_finished_per_minute": 17, + "requests_failed_per_minute": 0, + "request_total_duration": 3.57098, + "requests_total": 1, + "crawler_runtime": 3.59165 + }, + "file": { + "name": "_basic_crawler.py", + "path": "/crawlers/_basic/_basic_crawler.py" + }, + "function": "run", + "level": { "icon": "โ„น๏ธ", "name": "INFO", "no": 20 }, + "line": 583, + "message": "Final request statistics:", + "module": "_basic_crawler", + "name": "HttpCrawler", + "process": { "id": 198383, "name": "MainProcess" }, + "thread": { "id": 135312814966592, "name": "MainThread" }, + "time": { + "repr": "2025-03-17 17:14:45.339150+00:00", + "timestamp": 1742231685.33915 + } + } +} +``` diff --git a/docs/guides/service_locator.mdx b/docs/03_concepts/18_service_locator.mdx similarity index 91% rename from docs/guides/service_locator.mdx rename to docs/03_concepts/18_service_locator.mdx index fe10ce50c2..1f3d785111 100644 --- a/docs/guides/service_locator.mdx +++ b/docs/03_concepts/18_service_locator.mdx @@ -39,7 +39,7 @@ There are three core services that are managed by the service locator: `StorageClient` is the backend implementation for storages in Crawlee. It provides a unified interface for `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying storage implementation. Storage clients were already explained in the storage clients section. -Refer to the [Storage clients guide](./storage-clients) for more information about storage clients and how to use them. +Refer to the [Storage clients](./storage-clients) page for more information about storage clients and how to use them. ### EventManager @@ -129,8 +129,3 @@ Once a service has been retrieved from the service locator, attempting to set a {ServiceConflicts} -## Conclusion - -The `ServiceLocator` is a tool for managing global services in Crawlee. It provides a consistent way to configure and access services throughout the framework, ensuring that all components have access to the same configuration and services. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/code_examples/cookie_management/initial_cookies.py b/docs/03_concepts/code_examples/cookie_management/initial_cookies.py similarity index 100% rename from docs/guides/code_examples/cookie_management/initial_cookies.py rename to docs/03_concepts/code_examples/cookie_management/initial_cookies.py diff --git a/docs/guides/code_examples/cookie_management/persist_cookies.py b/docs/03_concepts/code_examples/cookie_management/persist_cookies.py similarity index 100% rename from docs/guides/code_examples/cookie_management/persist_cookies.py rename to docs/03_concepts/code_examples/cookie_management/persist_cookies.py diff --git a/docs/guides/code_examples/cookie_management/playwright_cookies.py b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py similarity index 100% rename from docs/guides/code_examples/cookie_management/playwright_cookies.py rename to docs/03_concepts/code_examples/cookie_management/playwright_cookies.py diff --git a/docs/guides/code_examples/cookie_management/read_write_cookies.py b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py similarity index 100% rename from docs/guides/code_examples/cookie_management/read_write_cookies.py rename to docs/03_concepts/code_examples/cookie_management/read_write_cookies.py diff --git a/docs/guides/code_examples/cookie_management/retry_pinned_session.py b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py similarity index 100% rename from docs/guides/code_examples/cookie_management/retry_pinned_session.py rename to docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py diff --git a/docs/guides/code_examples/cookie_management/retry_restore_cookies.py b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py similarity index 100% rename from docs/guides/code_examples/cookie_management/retry_restore_cookies.py rename to docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py diff --git a/docs/guides/code_examples/cookie_management/retry_single_session.py b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py similarity index 100% rename from docs/guides/code_examples/cookie_management/retry_single_session.py rename to docs/03_concepts/code_examples/cookie_management/retry_single_session.py diff --git a/docs/guides/code_examples/error_handling/change_handle_error_status.py b/docs/03_concepts/code_examples/error_handling/change_handle_error_status.py similarity index 100% rename from docs/guides/code_examples/error_handling/change_handle_error_status.py rename to docs/03_concepts/code_examples/error_handling/change_handle_error_status.py diff --git a/docs/guides/code_examples/error_handling/disable_retry.py b/docs/03_concepts/code_examples/error_handling/disable_retry.py similarity index 100% rename from docs/guides/code_examples/error_handling/disable_retry.py rename to docs/03_concepts/code_examples/error_handling/disable_retry.py diff --git a/docs/guides/code_examples/error_handling/handle_proxy_error.py b/docs/03_concepts/code_examples/error_handling/handle_proxy_error.py similarity index 100% rename from docs/guides/code_examples/error_handling/handle_proxy_error.py rename to docs/03_concepts/code_examples/error_handling/handle_proxy_error.py diff --git a/docs/examples/code_examples/parsel_crawler_with_error_snapshotter.py b/docs/03_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py similarity index 100% rename from docs/examples/code_examples/parsel_crawler_with_error_snapshotter.py rename to docs/03_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py diff --git a/docs/examples/code_examples/playwright_crawler_with_error_snapshotter.py b/docs/03_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py similarity index 100% rename from docs/examples/code_examples/playwright_crawler_with_error_snapshotter.py rename to docs/03_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py diff --git a/docs/guides/code_examples/http_clients/parsel_curl_impersonate_example.py b/docs/03_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py similarity index 100% rename from docs/guides/code_examples/http_clients/parsel_curl_impersonate_example.py rename to docs/03_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py diff --git a/docs/guides/code_examples/http_clients/parsel_httpx_example.py b/docs/03_concepts/code_examples/http_clients/parsel_httpx_example.py similarity index 100% rename from docs/guides/code_examples/http_clients/parsel_httpx_example.py rename to docs/03_concepts/code_examples/http_clients/parsel_httpx_example.py diff --git a/docs/guides/code_examples/http_clients/parsel_impit_example.py b/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py similarity index 100% rename from docs/guides/code_examples/http_clients/parsel_impit_example.py rename to docs/03_concepts/code_examples/http_clients/parsel_impit_example.py diff --git a/docs/guides/code_examples/running_in_web_server/__init__.py b/docs/03_concepts/code_examples/http_crawlers/__init__.py similarity index 100% rename from docs/guides/code_examples/running_in_web_server/__init__.py rename to docs/03_concepts/code_examples/http_crawlers/__init__.py diff --git a/docs/guides/code_examples/http_crawlers/beautifulsoup_example.py b/docs/03_concepts/code_examples/http_crawlers/beautifulsoup_example.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/beautifulsoup_example.py rename to docs/03_concepts/code_examples/http_crawlers/beautifulsoup_example.py diff --git a/docs/guides/code_examples/http_crawlers/custom_crawler_example.py b/docs/03_concepts/code_examples/http_crawlers/custom_crawler_example.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/custom_crawler_example.py rename to docs/03_concepts/code_examples/http_crawlers/custom_crawler_example.py diff --git a/docs/examples/code_examples/file_download.py b/docs/03_concepts/code_examples/http_crawlers/file_download.py similarity index 100% rename from docs/examples/code_examples/file_download.py rename to docs/03_concepts/code_examples/http_crawlers/file_download.py diff --git a/docs/examples/code_examples/file_download_stream.py b/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py similarity index 100% rename from docs/examples/code_examples/file_download_stream.py rename to docs/03_concepts/code_examples/http_crawlers/file_download_stream.py diff --git a/docs/guides/code_examples/http_crawlers/http_example.py b/docs/03_concepts/code_examples/http_crawlers/http_example.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/http_example.py rename to docs/03_concepts/code_examples/http_crawlers/http_example.py diff --git a/docs/guides/code_examples/http_crawlers/lexbor_parser.py b/docs/03_concepts/code_examples/http_crawlers/lexbor_parser.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/lexbor_parser.py rename to docs/03_concepts/code_examples/http_crawlers/lexbor_parser.py diff --git a/docs/guides/code_examples/http_crawlers/lxml_parser.py b/docs/03_concepts/code_examples/http_crawlers/lxml_parser.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/lxml_parser.py rename to docs/03_concepts/code_examples/http_crawlers/lxml_parser.py diff --git a/docs/guides/code_examples/http_crawlers/lxml_saxonche_parser.py b/docs/03_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/lxml_saxonche_parser.py rename to docs/03_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py diff --git a/docs/guides/code_examples/http_crawlers/parsel_example.py b/docs/03_concepts/code_examples/http_crawlers/parsel_example.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/parsel_example.py rename to docs/03_concepts/code_examples/http_crawlers/parsel_example.py diff --git a/docs/guides/code_examples/http_crawlers/pyquery_parser.py b/docs/03_concepts/code_examples/http_crawlers/pyquery_parser.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/pyquery_parser.py rename to docs/03_concepts/code_examples/http_crawlers/pyquery_parser.py diff --git a/docs/guides/code_examples/http_crawlers/scrapling_parser.py b/docs/03_concepts/code_examples/http_crawlers/scrapling_parser.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/scrapling_parser.py rename to docs/03_concepts/code_examples/http_crawlers/scrapling_parser.py diff --git a/docs/guides/code_examples/http_crawlers/selectolax_adaptive_run.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/selectolax_adaptive_run.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py diff --git a/docs/guides/code_examples/http_crawlers/selectolax_context.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_context.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/selectolax_context.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_context.py diff --git a/docs/guides/code_examples/http_crawlers/selectolax_crawler.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_crawler.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/selectolax_crawler.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_crawler.py diff --git a/docs/guides/code_examples/http_crawlers/selectolax_crawler_run.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_crawler_run.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/selectolax_crawler_run.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_crawler_run.py diff --git a/docs/guides/code_examples/http_crawlers/selectolax_parser.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_parser.py similarity index 100% rename from docs/guides/code_examples/http_crawlers/selectolax_parser.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_parser.py diff --git a/docs/guides/code_examples/http_headers/browser_page_headers.py b/docs/03_concepts/code_examples/http_headers/browser_page_headers.py similarity index 100% rename from docs/guides/code_examples/http_headers/browser_page_headers.py rename to docs/03_concepts/code_examples/http_headers/browser_page_headers.py diff --git a/docs/guides/code_examples/http_headers/set_headers.py b/docs/03_concepts/code_examples/http_headers/set_headers.py similarity index 100% rename from docs/guides/code_examples/http_headers/set_headers.py rename to docs/03_concepts/code_examples/http_headers/set_headers.py diff --git a/docs/examples/code_examples/configure_json_logging.py b/docs/03_concepts/code_examples/logging/configure_json_logging.py similarity index 100% rename from docs/examples/code_examples/configure_json_logging.py rename to docs/03_concepts/code_examples/logging/configure_json_logging.py diff --git a/docs/guides/code_examples/playwright_crawler/browser_configuration_example.py b/docs/03_concepts/code_examples/playwright_crawler/browser_configuration_example.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler/browser_configuration_example.py rename to docs/03_concepts/code_examples/playwright_crawler/browser_configuration_example.py diff --git a/docs/guides/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py b/docs/03_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py rename to docs/03_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py diff --git a/docs/guides/code_examples/playwright_crawler/browser_pool_page_hooks_example.py b/docs/03_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler/browser_pool_page_hooks_example.py rename to docs/03_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py diff --git a/docs/examples/code_examples/capture_screenshot_using_playwright.py b/docs/03_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py similarity index 100% rename from docs/examples/code_examples/capture_screenshot_using_playwright.py rename to docs/03_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py diff --git a/docs/guides/code_examples/playwright_crawler/multiple_launch_example.py b/docs/03_concepts/code_examples/playwright_crawler/multiple_launch_example.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler/multiple_launch_example.py rename to docs/03_concepts/code_examples/playwright_crawler/multiple_launch_example.py diff --git a/docs/guides/code_examples/playwright_crawler/navigation_hooks_example.py b/docs/03_concepts/code_examples/playwright_crawler/navigation_hooks_example.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler/navigation_hooks_example.py rename to docs/03_concepts/code_examples/playwright_crawler/navigation_hooks_example.py diff --git a/docs/examples/code_examples/playwright_block_requests.py b/docs/03_concepts/code_examples/playwright_crawler/playwright_block_requests.py similarity index 100% rename from docs/examples/code_examples/playwright_block_requests.py rename to docs/03_concepts/code_examples/playwright_crawler/playwright_block_requests.py diff --git a/docs/examples/code_examples/playwright_crawler.py b/docs/03_concepts/code_examples/playwright_crawler/playwright_crawler.py similarity index 100% rename from docs/examples/code_examples/playwright_crawler.py rename to docs/03_concepts/code_examples/playwright_crawler/playwright_crawler.py diff --git a/docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py b/docs/03_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler/plugin_browser_configuration_example.py rename to docs/03_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py diff --git a/docs/examples/code_examples/adaptive_playwright_crawler.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py similarity index 100% rename from docs/examples/code_examples/adaptive_playwright_crawler.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/handler.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/handler.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler_adaptive/handler.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/handler.py diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler_adaptive/init_parsel.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler_adaptive/init_prediction.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py diff --git a/docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py similarity index 100% rename from docs/guides/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py diff --git a/docs/guides/code_examples/proxy_management/inspecting_bs_example.py b/docs/03_concepts/code_examples/proxy_management/inspecting_bs_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/inspecting_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/inspecting_bs_example.py diff --git a/docs/guides/code_examples/proxy_management/inspecting_pw_example.py b/docs/03_concepts/code_examples/proxy_management/inspecting_pw_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/inspecting_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/inspecting_pw_example.py diff --git a/docs/guides/code_examples/proxy_management/integration_bs_example.py b/docs/03_concepts/code_examples/proxy_management/integration_bs_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/integration_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/integration_bs_example.py diff --git a/docs/guides/code_examples/proxy_management/integration_pw_example.py b/docs/03_concepts/code_examples/proxy_management/integration_pw_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/integration_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/integration_pw_example.py diff --git a/docs/guides/code_examples/proxy_management/quick_start_example.py b/docs/03_concepts/code_examples/proxy_management/quick_start_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/quick_start_example.py rename to docs/03_concepts/code_examples/proxy_management/quick_start_example.py diff --git a/docs/guides/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/session_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/session_bs_example.py diff --git a/docs/guides/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/session_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/session_pw_example.py diff --git a/docs/guides/code_examples/proxy_management/tiers_bs_example.py b/docs/03_concepts/code_examples/proxy_management/tiers_bs_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/tiers_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/tiers_bs_example.py diff --git a/docs/guides/code_examples/proxy_management/tiers_pw_example.py b/docs/03_concepts/code_examples/proxy_management/tiers_pw_example.py similarity index 100% rename from docs/guides/code_examples/proxy_management/tiers_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/tiers_pw_example.py diff --git a/docs/guides/code_examples/request_loaders/rl_basic_example.py b/docs/03_concepts/code_examples/request_loaders/rl_basic_example.py similarity index 100% rename from docs/guides/code_examples/request_loaders/rl_basic_example.py rename to docs/03_concepts/code_examples/request_loaders/rl_basic_example.py diff --git a/docs/guides/code_examples/request_loaders/rl_basic_example_with_persist.py b/docs/03_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py similarity index 100% rename from docs/guides/code_examples/request_loaders/rl_basic_example_with_persist.py rename to docs/03_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py diff --git a/docs/guides/code_examples/request_loaders/rl_tandem_example.py b/docs/03_concepts/code_examples/request_loaders/rl_tandem_example.py similarity index 100% rename from docs/guides/code_examples/request_loaders/rl_tandem_example.py rename to docs/03_concepts/code_examples/request_loaders/rl_tandem_example.py diff --git a/docs/guides/code_examples/request_loaders/rl_tandem_example_explicit.py b/docs/03_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py similarity index 100% rename from docs/guides/code_examples/request_loaders/rl_tandem_example_explicit.py rename to docs/03_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py diff --git a/docs/guides/code_examples/request_loaders/sitemap_basic_example.py b/docs/03_concepts/code_examples/request_loaders/sitemap_basic_example.py similarity index 100% rename from docs/guides/code_examples/request_loaders/sitemap_basic_example.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_basic_example.py diff --git a/docs/guides/code_examples/request_loaders/sitemap_example_with_persist.py b/docs/03_concepts/code_examples/request_loaders/sitemap_example_with_persist.py similarity index 100% rename from docs/guides/code_examples/request_loaders/sitemap_example_with_persist.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_example_with_persist.py diff --git a/docs/guides/code_examples/request_loaders/sitemap_tandem_example.py b/docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example.py similarity index 100% rename from docs/guides/code_examples/request_loaders/sitemap_tandem_example.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example.py diff --git a/docs/guides/code_examples/request_loaders/sitemap_tandem_example_explicit.py b/docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py similarity index 100% rename from docs/guides/code_examples/request_loaders/sitemap_tandem_example_explicit.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py diff --git a/docs/examples/code_examples/using_sitemap_request_loader.py b/docs/03_concepts/code_examples/request_loaders/using_sitemap_request_loader.py similarity index 100% rename from docs/examples/code_examples/using_sitemap_request_loader.py rename to docs/03_concepts/code_examples/request_loaders/using_sitemap_request_loader.py diff --git a/docs/guides/code_examples/request_router/adaptive_crawler_handlers.py b/docs/03_concepts/code_examples/request_router/adaptive_crawler_handlers.py similarity index 100% rename from docs/guides/code_examples/request_router/adaptive_crawler_handlers.py rename to docs/03_concepts/code_examples/request_router/adaptive_crawler_handlers.py diff --git a/docs/guides/code_examples/request_router/basic_request_handlers.py b/docs/03_concepts/code_examples/request_router/basic_request_handlers.py similarity index 100% rename from docs/guides/code_examples/request_router/basic_request_handlers.py rename to docs/03_concepts/code_examples/request_router/basic_request_handlers.py diff --git a/docs/guides/code_examples/request_router/custom_router_default_only.py b/docs/03_concepts/code_examples/request_router/custom_router_default_only.py similarity index 100% rename from docs/guides/code_examples/request_router/custom_router_default_only.py rename to docs/03_concepts/code_examples/request_router/custom_router_default_only.py diff --git a/docs/guides/code_examples/request_router/error_handler.py b/docs/03_concepts/code_examples/request_router/error_handler.py similarity index 100% rename from docs/guides/code_examples/request_router/error_handler.py rename to docs/03_concepts/code_examples/request_router/error_handler.py diff --git a/docs/guides/code_examples/request_router/failed_request_handler.py b/docs/03_concepts/code_examples/request_router/failed_request_handler.py similarity index 100% rename from docs/guides/code_examples/request_router/failed_request_handler.py rename to docs/03_concepts/code_examples/request_router/failed_request_handler.py diff --git a/docs/guides/code_examples/request_router/http_pre_navigation.py b/docs/03_concepts/code_examples/request_router/http_pre_navigation.py similarity index 100% rename from docs/guides/code_examples/request_router/http_pre_navigation.py rename to docs/03_concepts/code_examples/request_router/http_pre_navigation.py diff --git a/docs/guides/code_examples/request_router/playwright_pre_navigation.py b/docs/03_concepts/code_examples/request_router/playwright_pre_navigation.py similarity index 100% rename from docs/guides/code_examples/request_router/playwright_pre_navigation.py rename to docs/03_concepts/code_examples/request_router/playwright_pre_navigation.py diff --git a/docs/guides/code_examples/request_router/router_middleware.py b/docs/03_concepts/code_examples/request_router/router_middleware.py similarity index 100% rename from docs/guides/code_examples/request_router/router_middleware.py rename to docs/03_concepts/code_examples/request_router/router_middleware.py diff --git a/docs/guides/code_examples/request_router/simple_default_handler.py b/docs/03_concepts/code_examples/request_router/simple_default_handler.py similarity index 100% rename from docs/guides/code_examples/request_router/simple_default_handler.py rename to docs/03_concepts/code_examples/request_router/simple_default_handler.py diff --git a/docs/guides/code_examples/request_throttling/throttling_example.py b/docs/03_concepts/code_examples/request_throttling/throttling_example.py similarity index 100% rename from docs/guides/code_examples/request_throttling/throttling_example.py rename to docs/03_concepts/code_examples/request_throttling/throttling_example.py diff --git a/docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/03_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py similarity index 100% rename from docs/guides/code_examples/scaling_crawlers/max_tasks_per_minute_example.py rename to docs/03_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py diff --git a/docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/03_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py similarity index 100% rename from docs/guides/code_examples/scaling_crawlers/min_and_max_concurrency_example.py rename to docs/03_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py diff --git a/docs/guides/code_examples/service_locator/service_conflicts.py b/docs/03_concepts/code_examples/service_locator/service_conflicts.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_conflicts.py rename to docs/03_concepts/code_examples/service_locator/service_conflicts.py diff --git a/docs/guides/code_examples/service_locator/service_crawler_configuration.py b/docs/03_concepts/code_examples/service_locator/service_crawler_configuration.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_crawler_configuration.py rename to docs/03_concepts/code_examples/service_locator/service_crawler_configuration.py diff --git a/docs/guides/code_examples/service_locator/service_crawler_event_manager.py b/docs/03_concepts/code_examples/service_locator/service_crawler_event_manager.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_crawler_event_manager.py rename to docs/03_concepts/code_examples/service_locator/service_crawler_event_manager.py diff --git a/docs/guides/code_examples/service_locator/service_crawler_storage_client.py b/docs/03_concepts/code_examples/service_locator/service_crawler_storage_client.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_crawler_storage_client.py rename to docs/03_concepts/code_examples/service_locator/service_crawler_storage_client.py diff --git a/docs/guides/code_examples/service_locator/service_locator_configuration.py b/docs/03_concepts/code_examples/service_locator/service_locator_configuration.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_locator_configuration.py rename to docs/03_concepts/code_examples/service_locator/service_locator_configuration.py diff --git a/docs/guides/code_examples/service_locator/service_locator_event_manager.py b/docs/03_concepts/code_examples/service_locator/service_locator_event_manager.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_locator_event_manager.py rename to docs/03_concepts/code_examples/service_locator/service_locator_event_manager.py diff --git a/docs/guides/code_examples/service_locator/service_locator_storage_client.py b/docs/03_concepts/code_examples/service_locator/service_locator_storage_client.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_locator_storage_client.py rename to docs/03_concepts/code_examples/service_locator/service_locator_storage_client.py diff --git a/docs/guides/code_examples/service_locator/service_storage_configuration.py b/docs/03_concepts/code_examples/service_locator/service_storage_configuration.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_storage_configuration.py rename to docs/03_concepts/code_examples/service_locator/service_storage_configuration.py diff --git a/docs/guides/code_examples/service_locator/service_storage_storage_client.py b/docs/03_concepts/code_examples/service_locator/service_storage_storage_client.py similarity index 100% rename from docs/guides/code_examples/service_locator/service_storage_storage_client.py rename to docs/03_concepts/code_examples/service_locator/service_storage_storage_client.py diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py similarity index 100% rename from docs/guides/code_examples/session_management/multi_sessions_http.py rename to docs/03_concepts/code_examples/session_management/multi_sessions_http.py diff --git a/docs/guides/code_examples/session_management/one_session_http.py b/docs/03_concepts/code_examples/session_management/one_session_http.py similarity index 100% rename from docs/guides/code_examples/session_management/one_session_http.py rename to docs/03_concepts/code_examples/session_management/one_session_http.py diff --git a/docs/guides/code_examples/session_management/sm_basic.py b/docs/03_concepts/code_examples/session_management/sm_basic.py similarity index 100% rename from docs/guides/code_examples/session_management/sm_basic.py rename to docs/03_concepts/code_examples/session_management/sm_basic.py diff --git a/docs/guides/code_examples/session_management/sm_beautifulsoup.py b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py similarity index 100% rename from docs/guides/code_examples/session_management/sm_beautifulsoup.py rename to docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py diff --git a/docs/guides/code_examples/session_management/sm_http.py b/docs/03_concepts/code_examples/session_management/sm_http.py similarity index 100% rename from docs/guides/code_examples/session_management/sm_http.py rename to docs/03_concepts/code_examples/session_management/sm_http.py diff --git a/docs/guides/code_examples/session_management/sm_parsel.py b/docs/03_concepts/code_examples/session_management/sm_parsel.py similarity index 100% rename from docs/guides/code_examples/session_management/sm_parsel.py rename to docs/03_concepts/code_examples/session_management/sm_parsel.py diff --git a/docs/guides/code_examples/session_management/sm_playwright.py b/docs/03_concepts/code_examples/session_management/sm_playwright.py similarity index 100% rename from docs/guides/code_examples/session_management/sm_playwright.py rename to docs/03_concepts/code_examples/session_management/sm_playwright.py diff --git a/docs/guides/code_examples/session_management/sm_standalone.py b/docs/03_concepts/code_examples/session_management/sm_standalone.py similarity index 100% rename from docs/guides/code_examples/session_management/sm_standalone.py rename to docs/03_concepts/code_examples/session_management/sm_standalone.py diff --git a/docs/guides/code_examples/storage_clients/custom_storage_client_example.py b/docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/custom_storage_client_example.py rename to docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py diff --git a/docs/guides/code_examples/storage_clients/file_system_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/file_system_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py diff --git a/docs/guides/code_examples/storage_clients/file_system_storage_client_configuration_example.py b/docs/03_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/file_system_storage_client_configuration_example.py rename to docs/03_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py diff --git a/docs/guides/code_examples/storage_clients/memory_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/memory_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py diff --git a/docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py diff --git a/docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py b/docs/03_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py rename to docs/03_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py diff --git a/docs/guides/code_examples/storage_clients/registering_storage_clients_example.py b/docs/03_concepts/code_examples/storage_clients/registering_storage_clients_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/registering_storage_clients_example.py rename to docs/03_concepts/code_examples/storage_clients/registering_storage_clients_example.py diff --git a/docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py diff --git a/docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py b/docs/03_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py similarity index 100% rename from docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py rename to docs/03_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py diff --git a/docs/guides/code_examples/storages/cleaning_do_not_purge_example.py b/docs/03_concepts/code_examples/storages/cleaning_do_not_purge_example.py similarity index 100% rename from docs/guides/code_examples/storages/cleaning_do_not_purge_example.py rename to docs/03_concepts/code_examples/storages/cleaning_do_not_purge_example.py diff --git a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py b/docs/03_concepts/code_examples/storages/cleaning_purge_explicitly_example.py similarity index 100% rename from docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py rename to docs/03_concepts/code_examples/storages/cleaning_purge_explicitly_example.py diff --git a/docs/guides/code_examples/storages/dataset_basic_example.py b/docs/03_concepts/code_examples/storages/dataset_basic_example.py similarity index 100% rename from docs/guides/code_examples/storages/dataset_basic_example.py rename to docs/03_concepts/code_examples/storages/dataset_basic_example.py diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_example.py b/docs/03_concepts/code_examples/storages/dataset_with_crawler_example.py similarity index 100% rename from docs/guides/code_examples/storages/dataset_with_crawler_example.py rename to docs/03_concepts/code_examples/storages/dataset_with_crawler_example.py diff --git a/docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py b/docs/03_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py similarity index 100% rename from docs/guides/code_examples/storages/dataset_with_crawler_explicit_example.py rename to docs/03_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_csv.py b/docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py similarity index 100% rename from docs/examples/code_examples/export_entire_dataset_to_file_csv.py rename to docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py diff --git a/docs/examples/code_examples/export_entire_dataset_to_file_json.py b/docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_json.py similarity index 100% rename from docs/examples/code_examples/export_entire_dataset_to_file_json.py rename to docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_json.py diff --git a/docs/guides/code_examples/storages/helper_add_requests_example.py b/docs/03_concepts/code_examples/storages/helper_add_requests_example.py similarity index 100% rename from docs/guides/code_examples/storages/helper_add_requests_example.py rename to docs/03_concepts/code_examples/storages/helper_add_requests_example.py diff --git a/docs/guides/code_examples/storages/helper_enqueue_links_example.py b/docs/03_concepts/code_examples/storages/helper_enqueue_links_example.py similarity index 100% rename from docs/guides/code_examples/storages/helper_enqueue_links_example.py rename to docs/03_concepts/code_examples/storages/helper_enqueue_links_example.py diff --git a/docs/guides/code_examples/storages/kvs_basic_example.py b/docs/03_concepts/code_examples/storages/kvs_basic_example.py similarity index 100% rename from docs/guides/code_examples/storages/kvs_basic_example.py rename to docs/03_concepts/code_examples/storages/kvs_basic_example.py diff --git a/docs/guides/code_examples/storages/kvs_with_crawler_example.py b/docs/03_concepts/code_examples/storages/kvs_with_crawler_example.py similarity index 100% rename from docs/guides/code_examples/storages/kvs_with_crawler_example.py rename to docs/03_concepts/code_examples/storages/kvs_with_crawler_example.py diff --git a/docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py b/docs/03_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py similarity index 100% rename from docs/guides/code_examples/storages/kvs_with_crawler_explicit_example.py rename to docs/03_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py diff --git a/docs/guides/code_examples/storages/opening.py b/docs/03_concepts/code_examples/storages/opening.py similarity index 100% rename from docs/guides/code_examples/storages/opening.py rename to docs/03_concepts/code_examples/storages/opening.py diff --git a/docs/guides/code_examples/storages/rq_basic_example.py b/docs/03_concepts/code_examples/storages/rq_basic_example.py similarity index 100% rename from docs/guides/code_examples/storages/rq_basic_example.py rename to docs/03_concepts/code_examples/storages/rq_basic_example.py diff --git a/docs/guides/code_examples/storages/rq_with_crawler_example.py b/docs/03_concepts/code_examples/storages/rq_with_crawler_example.py similarity index 100% rename from docs/guides/code_examples/storages/rq_with_crawler_example.py rename to docs/03_concepts/code_examples/storages/rq_with_crawler_example.py diff --git a/docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py b/docs/03_concepts/code_examples/storages/rq_with_crawler_explicit_example.py similarity index 100% rename from docs/guides/code_examples/storages/rq_with_crawler_explicit_example.py rename to docs/03_concepts/code_examples/storages/rq_with_crawler_explicit_example.py diff --git a/docs/04_guides/01_crawling_links.mdx b/docs/04_guides/01_crawling_links.mdx new file mode 100644 index 0000000000..bc98f501f1 --- /dev/null +++ b/docs/04_guides/01_crawling_links.mdx @@ -0,0 +1,141 @@ +--- +id: crawling-links +title: Crawling links +description: How to crawl a list of URLs, discover and enqueue new links on the fly, and control which links get followed. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import MultipleUrlsBeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_multiple_urls_bs.py'; +import MultipleUrlsPlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_multiple_urls_pw.py'; + +import AllLinksBeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_all_links_on_website_bs.py'; +import AllLinksPlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_all_links_on_website_pw.py'; + +import SpecificLinksBeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_specific_links_on_website_bs.py'; +import SpecificLinksPlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_specific_links_on_website_pw.py'; + +import ExtractAndAddBeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py'; +import ExtractAndAddPlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py'; + +import StrategyAllLinksExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_website_with_relative_links_all_links.py'; +import StrategySameDomainExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py'; +import StrategySameHostnameExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py'; +import StrategySameOriginExample from '!!raw-loader!roa-loader!./code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py'; + +Most crawls start from a handful of URLs and grow by following links discovered on the visited pages. This guide shows how to crawl a fixed list of URLs, how to enqueue newly discovered links with the `enqueue_links` helper, and how to control which links end up in the `RequestQueue`. + +## Crawling a list of URLs + +The simplest case is crawling a known list of URLs. Pass the list to `crawler.run` and the request handler is invoked for each of them. + + + + + {MultipleUrlsBeautifulSoupExample} + + + + + {MultipleUrlsPlaywrightExample} + + + + +## Crawling all links on a website + +To crawl a whole website, call the `enqueue_links` helper in the request handler. It finds the links on the current page and adds them to the `RequestQueue`, so the crawler systematically works through the site page by page. + +:::tip + +If no options are given, by default the method will only add links that are under the same hostname. This behavior can be controlled with the `strategy` option. For details, see [Enqueue strategies](#enqueue-strategies). + +::: + + + + + {AllLinksBeautifulSoupExample} + + + + + {AllLinksPlaywrightExample} + + + + +## Crawling specific links + +You'll often want to follow only links that match certain patterns. Pass the `include` or `exclude` parameters to `enqueue_links` - both accept lists of globs or regular expressions - and only the matching links are added to the `RequestQueue`. This keeps the crawl focused on the relevant sections of a website and avoids scraping unnecessary content. + + + + + {SpecificLinksBeautifulSoupExample} + + + + + {SpecificLinksPlaywrightExample} + + + + +### Even more control over the enqueued links + +`enqueue_links` is a convenience helper. Internally it calls `extract_links` to find the links and `add_requests` to add them to the queue. If you need custom filtering of the extracted links before enqueuing them, use `extract_links` and `add_requests` directly: + + + + + {ExtractAndAddBeautifulSoupExample} + + + + + {ExtractAndAddPlaywrightExample} + + + + +## Enqueue strategies + +`enqueue_links` automatically resolves relative links based on the page's context and decides which of them to follow according to the `strategy` option (the `EnqueueStrategy` type alias). Four strategies are available: + +- `all` - Enqueues all links found, regardless of the domain they point to. Useful when you want to follow every link, including those that navigate to external websites. +- `same-domain` - Enqueues all links found that share the same domain name, including any possible subdomains. +- `same-hostname` - Enqueues all links found for the exact same hostname. This is the **default** strategy. It restricts the crawl to links with the same hostname as the current page and excludes subdomains. +- `same-origin` - Enqueues all links found that share the same origin - the same protocol, domain, and port - ensuring a strict scope for the crawl. + +:::note + +These examples use the `BeautifulSoupCrawler`, but the same method is available for all crawlers and works exactly the same way. + +::: + + + + + {StrategyAllLinksExample} + + + + + {StrategySameDomainExample} + + + + + {StrategySameHostnameExample} + + + + + {StrategySameOriginExample} + + + diff --git a/docs/examples/fill_and_submit_web_form.mdx b/docs/04_guides/02_fill_and_submit_web_form.mdx similarity index 95% rename from docs/examples/fill_and_submit_web_form.mdx rename to docs/04_guides/02_fill_and_submit_web_form.mdx index e81af4623f..95a917da75 100644 --- a/docs/examples/fill_and_submit_web_form.mdx +++ b/docs/04_guides/02_fill_and_submit_web_form.mdx @@ -1,6 +1,7 @@ --- id: fill-and-submit-web-form -title: Fill and submit web form +title: Filling and submitting web forms +description: How to inspect a web form, prepare a POST request from its fields, and submit it with an HTTP crawler. --- import ApiLink from '@site/src/components/ApiLink'; @@ -8,9 +9,9 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RequestExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_request.py'; -import CrawlerExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_crawler.py'; -import AutomatedExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form_automated.py'; +import RequestExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py'; +import CrawlerExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py'; +import AutomatedExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py'; This example demonstrates how to fill and submit a web form using the `HttpCrawler` crawler. The same approach applies to any crawler that inherits from it, such as the `BeautifulSoupCrawler` or `ParselCrawler`. These two crawlers can also [fill in the form automatically](#fill-in-the-form-automatically). diff --git a/docs/guides/crawler_login.mdx b/docs/04_guides/03_crawler_login.mdx similarity index 57% rename from docs/guides/crawler_login.mdx rename to docs/04_guides/03_crawler_login.mdx index 3dcdc7c72d..197a017f89 100644 --- a/docs/guides/crawler_login.mdx +++ b/docs/04_guides/03_crawler_login.mdx @@ -5,16 +5,19 @@ description: How to log in to websites with Crawlee. --- import ApiLink from '@site/src/components/ApiLink'; +import CodeBlock from '@theme/CodeBlock'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import PlaywrightLogin from '!!raw-loader!roa-loader!./code_examples/login_crawler/playwright_login.py'; import HttpLogin from '!!raw-loader!roa-loader!./code_examples/login_crawler/http_login.py'; +import ChromeProfileExample from '!!raw-loader!./code_examples/login_crawler/using_browser_profiles_chrome.py'; +import FirefoxProfileExample from '!!raw-loader!./code_examples/login_crawler/using_browser_profiles_firefox.py'; Many websites require authentication to access their content. This guide demonstrates how to implement login functionality using both `PlaywrightCrawler` and `HttpCrawler`. ## Session management for authentication -When implementing authentication, you'll typically want to maintain the same `Session` throughout your crawl to preserve login state. This requires proper configuration of the `SessionPool`. For more details, see our [session management guide](./session-management). To work with the cookies themselves, see [Cookie management](./cookie-management). +When implementing authentication, you'll typically want to maintain the same `Session` throughout your crawl to preserve login state. This requires proper configuration of the `SessionPool`. For more details, see the [Session management](../concepts/session-management) page. To work with the cookies themselves, see [Cookie management](../concepts/cookie-management). If your use case requires multiple authenticated sessions with different credentials, you can: @@ -40,3 +43,33 @@ HTTP-based authentication often varies significantly between websites. Using bro {HttpLogin} + +## Using browser profiles + +Instead of automating the login flow, you can run `PlaywrightCrawler` with your local browser profile from [Chrome](https://www.google.com/intl/us/chrome/) or [Firefox](https://www.firefox.com/). Browser profiles carry existing login sessions, saved passwords, and other personalized browser data, which lets the crawler access content that requires authentication without logging in programmatically. + +### Chrome + +To run the crawler with your Chrome profile, you need to know the path to your profile files. You can find it by entering `chrome://version/` as a URL in your Chrome browser. If you have multiple profiles, pay attention to the profile name - with a single profile, it's always `Default`. + +:::warning Profile access limitation + +Due to [Chrome's security policies](https://developer.chrome.com/blog/remote-debugging-port), automation cannot use your main browsing profile directly. The example copies your profile to a temporary location as a workaround. + +::: + +Make sure you don't have any running Chrome browser processes before running this code: + + + {ChromeProfileExample} + + +### Firefox + +To find the path to your Firefox profile, enter `about:profiles` as a URL in your Firefox browser. Unlike Chrome, you can use your standard profile path directly without copying it first. + +Make sure you don't have any running Firefox browser processes before running this code: + + + {FirefoxProfileExample} + diff --git a/docs/guides/avoid_blocking.mdx b/docs/04_guides/04_avoid_blocking.mdx similarity index 70% rename from docs/guides/avoid_blocking.mdx rename to docs/04_guides/04_avoid_blocking.mdx index f800d11c45..2204cd2520 100644 --- a/docs/guides/avoid_blocking.mdx +++ b/docs/04_guides/04_avoid_blocking.mdx @@ -1,7 +1,7 @@ --- id: avoid-blocking title: Avoid getting blocked -description: How to avoid getting blocked when scraping +description: How to avoid getting blocked by anti-bot protections, using browser fingerprints and stealth browsers. --- import ApiLink from '@site/src/components/ApiLink'; @@ -9,12 +9,12 @@ import CodeBlock from '@theme/CodeBlock'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import PlaywrightDefaultFingerprintGenerator from '!!raw-loader!roa-loader!./code_examples/avoid_blocking/playwright_with_fingerprint_generator.py'; -import PlaywrightWithCamoufox from '!!raw-loader!roa-loader!../examples/code_examples/playwright_crawler_with_camoufox.py'; +import PlaywrightWithCamoufox from '!!raw-loader!roa-loader!./code_examples/avoid_blocking/playwright_crawler_with_camoufox.py'; import PlaywrightWithCloakBrowser from '!!raw-loader!roa-loader!./code_examples/avoid_blocking/playwright_with_cloakbrowser.py'; import PlaywrightDefaultFingerprintGeneratorWithArgs from '!!raw-loader!./code_examples/avoid_blocking/default_fingerprint_generator_with_args.py'; -A scraper might get blocked for numerous reasons. Let's narrow it down to the two main ones. The first is a bad or blocked IP address. You can learn about this topic in the [proxy management guide](./proxy-management). The second reason is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) (or signatures), which we will explore more in this guide. Check the [Apify Academy anti-scraping course](https://docs.apify.com/academy/anti-scraping) to gain a deeper theoretical understanding of blocking and learn a few tips and tricks. +A scraper might get blocked for numerous reasons. Let's narrow it down to the two main ones. The first is a bad or blocked IP address. You can learn about this topic on the [Proxy management](../concepts/proxy-management) page. The second reason is [browser fingerprints](https://pixelprivacy.com/resources/browser-fingerprinting/) (or signatures), which we will explore more in this guide. Check the [Apify Academy anti-scraping course](https://docs.apify.com/academy/anti-scraping) to gain a deeper theoretical understanding of blocking and learn a few tips and tricks. Browser fingerprint is a collection of browser attributes and significant features that can show if our browser is a bot or a real user. Moreover, most browsers have these unique features that allow the website to track the browser even within different IP addresses. This is the main reason why scrapers should change browser fingerprints while doing browser-based scraping. In return, it should significantly reduce the blocking. @@ -36,7 +36,15 @@ If you do not want to use fingerprints, then pass `fingerprint_generator=None` a ## Using Camoufox -In some cases even `PlaywrightCrawler` with fingerprints is not enough. You can try using `PlaywrightCrawler` together with [Camoufox](https://camoufox.com/). See the example integration below: +In some cases even `PlaywrightCrawler` with fingerprints is not enough. You can try using `PlaywrightCrawler` together with [Camoufox](https://camoufox.com/), a stealthy, minimalistic build of Firefox. The example below integrates Camoufox into `PlaywrightCrawler` using `BrowserPool` with a custom `PlaywrightBrowserPlugin`. + +Camoufox is an external tool and is not part of Crawlee, so [install it separately](https://pypi.org/project/camoufox/). Note that Camoufox uses a custom build of Firefox which can be hundreds of MB large. You can pre-download it with `python3 -m camoufox fetch`. Otherwise Camoufox downloads it automatically on first run. See the [Camoufox Python interface documentation](https://github.com/daijro/camoufox/tree/main/pythonlib#camoufox-python-interface) for details. + +:::tip + +You can generate a project with the Camoufox integration already set up through the Crawlee CLI. Run `crawlee create` and pick `Playwright-camoufox` when asked for the crawler type. + +::: {PlaywrightWithCamoufox} diff --git a/docs/examples/respect_robots_txt_file.mdx b/docs/04_guides/05_respect_robots_txt_file.mdx similarity index 86% rename from docs/examples/respect_robots_txt_file.mdx rename to docs/04_guides/05_respect_robots_txt_file.mdx index dc509e16b8..84f2f714d2 100644 --- a/docs/examples/respect_robots_txt_file.mdx +++ b/docs/04_guides/05_respect_robots_txt_file.mdx @@ -1,13 +1,14 @@ --- id: respect-robots-txt-file -title: Respect robots.txt file +title: Respecting robots.txt +description: How to make a crawler respect the rules websites declare for bots in their robots.txt file. --- import ApiLink from '@site/src/components/ApiLink'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RespectRobotsTxt from '!!raw-loader!roa-loader!./code_examples/respect_robots_txt_file.py'; -import OnSkippedRequest from '!!raw-loader!roa-loader!./code_examples/respect_robots_on_skipped_request.py'; +import RespectRobotsTxt from '!!raw-loader!roa-loader!./code_examples/respect_robots_txt_file/respect_robots_txt_file.py'; +import OnSkippedRequest from '!!raw-loader!roa-loader!./code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py'; This example demonstrates how to configure your crawler to respect the rules established by websites for crawlers as described in the [robots.txt](https://www.robotstxt.org/robotstxt.html) file. diff --git a/docs/04_guides/06_stopping_and_resuming.mdx b/docs/04_guides/06_stopping_and_resuming.mdx new file mode 100644 index 0000000000..2a05f28420 --- /dev/null +++ b/docs/04_guides/06_stopping_and_resuming.mdx @@ -0,0 +1,56 @@ +--- +id: stopping-and-resuming-crawlers +title: Stopping and resuming crawlers +description: How to stop a crawler from a request handler, keep it alive while waiting for more requests, and resume an interrupted crawl. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import CrawlerStopExample from '!!raw-loader!roa-loader!./code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py'; +import CrawlerKeepAliveExample from '!!raw-loader!roa-loader!./code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py'; +import ResumeCrawlExample from '!!raw-loader!roa-loader!./code_examples/stopping_and_resuming/resuming_paused_crawl.py'; + +A crawler normally runs until its request queue is empty. This guide covers the cases where you want to change that: stopping the crawl early once you've found what you're looking for, keeping the crawler alive while it waits for more requests, and resuming an interrupted crawl from where it stopped. + +All the options on this page are available to every crawler that inherits from `BasicCrawler`. The examples use `BeautifulSoupCrawler`, but they work the same with any other crawler. + +## Stopping a crawler + +Call `crawler.stop()` to stop the crawler, for example once it finds what it's looking for. The crawler won't pick up any new requests, but requests that are already being processed concurrently are finished. The optional `reason` argument is a string that appears in the logs, which can improve their readability, especially when multiple conditions can trigger the stop. + + + {CrawlerStopExample} + + +## Keeping a crawler alive + +To keep the crawler running even when there are no requests to process at the moment, pass `keep_alive=True` to the crawler's constructor (see `BasicCrawlerOptions`). The crawler then waits for more requests to be added instead of finishing. This setup is useful when another component adds requests over time, as in the [Running in web server](./running-in-web-server) guide. A crawler started with `keep_alive=True` is stopped by calling `crawler.stop()`. + + + {CrawlerKeepAliveExample} + + +## Resuming a paused crawl + +When a local crawler run is interrupted - say with `CTRL+C` - the default behavior is to start over on the next run, because Crawlee purges the storage on start. To continue from the previous state instead, disable `purge_on_start` in the `Configuration`. + +Use the following code and perform two sequential runs. During the first run, stop the crawler by pressing `CTRL+C`, and the second run will resume crawling from where it stopped. + + + {ResumeCrawlExample} + + +Perform the first run, interrupting the crawler with `CTRL+C` after 2 links have been processed. + +![Run with interruption](/img/resuming-paused-crawl/00.webp 'Run with interruption.') + +Now resume crawling after the pause to process the remaining 3 links. + +![Resuming crawling](/img/resuming-paused-crawl/01.webp 'Resuming crawling.') + +Alternatively, set the environment variable `CRAWLEE_PURGE_ON_START=0` instead of `configuration.purge_on_start = False`: + +```bash +CRAWLEE_PURGE_ON_START=0 python your_crawler.py +``` diff --git a/docs/examples/run_parallel_crawlers.mdx b/docs/04_guides/07_run_parallel_crawlers.mdx similarity index 76% rename from docs/examples/run_parallel_crawlers.mdx rename to docs/04_guides/07_run_parallel_crawlers.mdx index fba5c437b7..345eca8fd3 100644 --- a/docs/examples/run_parallel_crawlers.mdx +++ b/docs/04_guides/07_run_parallel_crawlers.mdx @@ -1,18 +1,19 @@ --- id: run-parallel-crawlers -title: Run parallel crawlers +title: Running crawlers in parallel +description: How to run two crawlers in parallel and pass links between them through request queues. --- import ApiLink from '@site/src/components/ApiLink'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import RunParallelCrawlersExample from '!!raw-loader!roa-loader!./code_examples/run_parallel_crawlers.py'; +import RunParallelCrawlersExample from '!!raw-loader!roa-loader!./code_examples/run_parallel_crawlers/run_parallel_crawlers.py'; This example demonstrates how to run two parallel crawlers where one crawler processes links discovered by another crawler. -In some situations, you may need different approaches for scraping data from a website. For example, you might use `PlaywrightCrawler` for navigating JavaScript-heavy pages and a faster, more lightweight `ParselCrawler` for processing static pages. One way to solve this is to use `AdaptivePlaywrightCrawler`, see the [Adaptive Playwright crawler example](./adaptive-playwright-crawler) to learn more. +In some situations, you may need different approaches for scraping data from a website. For example, you might use `PlaywrightCrawler` for navigating JavaScript-heavy pages and a faster, more lightweight `ParselCrawler` for processing static pages. One way to solve this is to use `AdaptivePlaywrightCrawler`, see the [Adaptive Playwright crawler](../concepts/adaptive-playwright-crawler) page to learn more. -The code below demonstrates an alternative approach using two separate crawlers. Links are passed between crawlers via `RequestQueue` aliases. The `keep_alive` option allows the Playwright crawler to run in the background and wait for incoming links without stopping when its queue is empty. You can also use different storage clients for each crawler without losing the ability to pass links between queues. Learn more about available storage clients in this [guide](/python/docs/guides/storage-clients). +The code below demonstrates an alternative approach using two separate crawlers. Links are passed between crawlers via `RequestQueue` aliases. The `keep_alive` option allows the Playwright crawler to run in the background and wait for incoming links without stopping when its queue is empty. You can also use different storage clients for each crawler without losing the ability to pass links between queues. Learn more about available storage clients on the [Storage clients](../concepts/storage-clients) page. {RunParallelCrawlersExample} diff --git a/docs/guides/running_in_web_server.mdx b/docs/04_guides/08_running_in_web_server.mdx similarity index 96% rename from docs/guides/running_in_web_server.mdx rename to docs/04_guides/08_running_in_web_server.mdx index 8e29bf4cb7..be3f1465b1 100644 --- a/docs/guides/running_in_web_server.mdx +++ b/docs/04_guides/08_running_in_web_server.mdx @@ -1,7 +1,7 @@ --- id: running-in-web-server title: Running in web server -description: Running in web server +description: How to run a crawler inside a web server and return crawl results in response to HTTP requests. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/guides/trace_and_monitor_crawlers.mdx b/docs/04_guides/09_trace_and_monitor_crawlers.mdx similarity index 100% rename from docs/guides/trace_and_monitor_crawlers.mdx rename to docs/04_guides/09_trace_and_monitor_crawlers.mdx diff --git a/docs/guides/creating_web_archive.mdx b/docs/04_guides/10_creating_web_archive.mdx similarity index 98% rename from docs/guides/creating_web_archive.mdx rename to docs/04_guides/10_creating_web_archive.mdx index bafc8ec888..6040d78d80 100644 --- a/docs/guides/creating_web_archive.mdx +++ b/docs/04_guides/10_creating_web_archive.mdx @@ -1,7 +1,7 @@ --- id: creating-web-archive title: Creating web archive -description: How to create a Web ARChive (WARC) with Crawlee +description: How to create a Web ARChive (WARC) of the crawled pages with Crawlee. --- import ApiLink from '@site/src/components/ApiLink'; diff --git a/docs/guides/secure_scraping.mdx b/docs/04_guides/11_secure_scraping.mdx similarity index 95% rename from docs/guides/secure_scraping.mdx rename to docs/04_guides/11_secure_scraping.mdx index 3cfa3097d6..97b9b07279 100644 --- a/docs/guides/secure_scraping.mdx +++ b/docs/04_guides/11_secure_scraping.mdx @@ -42,7 +42,7 @@ await context.enqueue_links() await context.enqueue_links(strategy='all') ``` -The same rules apply to URLs read by a `SitemapRequestLoader` and to `Sitemap:` directives in `robots.txt`. A target can't seed your queue with off-host URLs that way either. Following `robots.txt` rules is itself opt-in: `respect_robots_txt_file` is `False` by default. The [respecting robots.txt example](../examples/respect-robots-txt-file) shows how to turn it on. +The same rules apply to URLs read by a `SitemapRequestLoader` and to `Sitemap:` directives in `robots.txt`. A target can't seed your queue with off-host URLs that way either. Following `robots.txt` rules is itself opt-in: `respect_robots_txt_file` is `False` by default. The [respecting robots.txt example](./respect-robots-txt-file) shows how to turn it on. ### Widening the scope @@ -79,7 +79,7 @@ It's the same rule in every case. Treat an extracted value as untrusted until yo ## Untrusted proxies -Crawlers route requests through [proxies](./proxy-management) to rotate IPs and avoid blocking. A proxy sees and can alter every request and response that passes through it, so a free, unknown, or compromised proxy is a man-in-the-middle. It can log the URLs and data you send, read and modify responses, and capture access credentials like login submissions, API keys, `Authorization` headers, and session cookies. An attacker who collects those can reuse them to take over the accounts behind them. +Crawlers route requests through [proxies](../concepts/proxy-management) to rotate IPs and avoid blocking. A proxy sees and can alter every request and response that passes through it, so a free, unknown, or compromised proxy is a man-in-the-middle. It can log the URLs and data you send, read and modify responses, and capture access credentials like login submissions, API keys, `Authorization` headers, and session cookies. An attacker who collects those can reuse them to take over the accounts behind them. The built-in HTTP clients verify TLS certificates by default. Keeping that on and keeping traffic on HTTPS stops a proxy from reading or rewriting payloads, though it still sees the destination host through SNI and `CONNECT`. Prefer proxy providers you trust. Don't send credentials or secrets through a proxy you don't control. Don't disable certificate verification just to make an untrusted proxy work. diff --git a/docs/guides/extending_crawlee.mdx b/docs/04_guides/12_extending_crawlee.mdx similarity index 93% rename from docs/guides/extending_crawlee.mdx rename to docs/04_guides/12_extending_crawlee.mdx index 6e4a3d3977..07c4b7144f 100644 --- a/docs/guides/extending_crawlee.mdx +++ b/docs/04_guides/12_extending_crawlee.mdx @@ -124,7 +124,7 @@ The context exposes the parsed data to handlers, and the crawler ties the parser Browser crawlers use the same orchestration with a browser-backed context. Extend `PlaywrightCrawler` when an integration needs crawler-level browser behavior or a different handler context. `StagehandCrawler` is an example. It extends `PlaywrightCrawler` with a Stagehand-specific context and browser behavior. If only browser launch or lifecycle differs, a browser plugin is the narrower extension point. -See the [HTTP crawlers guide](./http-crawlers) for a worked example built on `selectolax`, and the [Stagehand crawler guide](./stagehand-crawler) for `StagehandCrawler` itself. The [Architecture overview](./architecture-overview) explains how HTTP and browser crawlers relate to the other components. +See the [HTTP crawlers](../concepts/http-crawlers) page for a worked example built on `selectolax`, and the [Stagehand crawler guide](./stagehand-crawler) for `StagehandCrawler` itself. The [Architecture overview](../concepts/architecture-overview) explains how HTTP and browser crawlers relate to the other components. ### HTTP clients @@ -139,7 +139,7 @@ The contract is `HttpClient`: Crawlee ships `ImpitHttpClient`, `HttpxHttpClient`, and `CurlImpersonateHttpClient`. -See the [HTTP clients guide](./http-clients) for the libraries these clients wrap and their installation requirements. +See the [HTTP clients](../concepts/http-clients) page for the libraries these clients wrap and their installation requirements. ### Storage clients @@ -153,7 +153,7 @@ The `StorageClient` contract defines A custom backend implements all four classes. -See the [Storage clients guide](./storage-clients) for the built-in implementations and a custom client example. +See the [Storage clients](../concepts/storage-clients) page for the built-in implementations and a custom client example. ### Browser plugins @@ -170,7 +170,7 @@ Implement this base contract directly when the launch and lifecycle are too spec Most integrations should start with `PlaywrightBrowserPlugin`. Configure it when its launch and context options cover the required browser. Extend it when you need a custom Playwright-compatible launch path while preserving its standard lifecycle and context handling. -See the [Playwright crawler guide](./playwright-crawler) for the responsibilities a subclass has to preserve, and the [Camoufox example](../examples/playwright-crawler-with-camoufox) for a complete integration. +See the [Playwright crawler](../concepts/playwright-crawler) page for the responsibilities a subclass has to preserve, and the [Camoufox example](./avoid-blocking#using-camoufox) for a complete integration. ## Choosing an extension point diff --git a/docs/guides/pydantic_ai_crawler.mdx b/docs/04_guides/13_pydantic_ai_crawler.mdx similarity index 100% rename from docs/guides/pydantic_ai_crawler.mdx rename to docs/04_guides/13_pydantic_ai_crawler.mdx diff --git a/docs/guides/stagehand_crawler.mdx b/docs/04_guides/14_stagehand_crawler.mdx similarity index 97% rename from docs/guides/stagehand_crawler.mdx rename to docs/04_guides/14_stagehand_crawler.mdx index adbd31b3e9..7bc7cab98b 100644 --- a/docs/guides/stagehand_crawler.mdx +++ b/docs/04_guides/14_stagehand_crawler.mdx @@ -29,7 +29,7 @@ Use `StagehandCrawler` when: - **Interactions are complex** - multi-step forms, dynamic menus, or context-dependent flows that are hard to script. - **Rapid prototyping** - you want to build a scraper quickly without spending time reverse-engineering the page structure. -For straightforward scraping tasks where the page structure is stable and well-known, `PlaywrightCrawler` is more efficient, read more in that [guide](./playwright-crawler). +For straightforward scraping tasks where the page structure is stable and well-known, `PlaywrightCrawler` is more efficient, read more on the [Playwright crawler](../concepts/playwright-crawler) page. ## Installation diff --git a/docs/guides/scrapy_migration.mdx b/docs/04_guides/15_scrapy_migration.mdx similarity index 92% rename from docs/guides/scrapy_migration.mdx rename to docs/04_guides/15_scrapy_migration.mdx index a5e4c29a9e..b7d03ad9c0 100644 --- a/docs/guides/scrapy_migration.mdx +++ b/docs/04_guides/15_scrapy_migration.mdx @@ -37,7 +37,7 @@ import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/scrapy_mi [Scrapy](https://scrapy.org/) and Crawlee solve the same problem, so most of what you know carries over. The concepts line up almost one to one, and the selector code often moves across without changes. Scrapy parses HTML with [Parsel](https://parsel.readthedocs.io/), and so does Crawlee's `ParselCrawler`. Your `response.css()` and `response.xpath()` queries keep working on `context.selector`. -This guide maps Scrapy concepts to their Crawlee equivalents and rewrites a small spider step by step. The examples use `ParselCrawler` because it's the closest match for a Scrapy spider. The other HTTP option is `BeautifulSoupCrawler`, and the choice between them is covered in the [HTTP crawlers guide](./http-crawlers). For pages that need a browser, Crawlee has `PlaywrightCrawler`, covered in the [Playwright crawler guide](./playwright-crawler). +This guide maps Scrapy concepts to their Crawlee equivalents and rewrites a small spider step by step. The examples use `ParselCrawler` because it's the closest match for a Scrapy spider. The other HTTP option is `BeautifulSoupCrawler`, and the choice between them is covered on the [HTTP crawlers](../concepts/http-crawlers) page. For pages that need a browser, Crawlee has `PlaywrightCrawler`, covered on the [Playwright crawler](../concepts/playwright-crawler) page. The Crawlee examples cap each run with `max_requests_per_crawl` to keep the demos short. The Scrapy equivalent is the `CLOSESPIDER_PAGECOUNT` setting from the built-in `CloseSpider` extension. @@ -123,7 +123,7 @@ Note that: ## Routing with labels -Scrapy routes pages by passing a `callback` to each request. A listing page hands its links to `parse_author`. In Crawlee you attach a `label` to the enqueued requests, then register a handler for that label. Each request goes to the matching handler, and unlabeled requests fall through to the default one. For details, see the [Request router guide](./request-router). +Scrapy routes pages by passing a `callback` to each request. A listing page hands its links to `parse_author`. In Crawlee you attach a `label` to the enqueued requests, then register a handler for that label. Each request goes to the matching handler, and unlabeled requests fall through to the default one. For details, see the [Request router](../concepts/request-router) page. @@ -191,11 +191,11 @@ Scrapy writes results through the `FEEDS` setting or the `-O` command-line flag. -The default (unnamed) `Dataset` stays on disk after the run finishes, but the next run clears it before starting, since `Configuration.purge_on_start` defaults to `True`. To keep data across runs, use a named dataset or turn that option off. For details, see the [Storages guide](./storages). To continue an interrupted crawl from where it stopped, see the [Resuming a paused crawl](../examples/resuming-paused-crawl) example. +The default (unnamed) `Dataset` stays on disk after the run finishes, but the next run clears it before starting, since `Configuration.purge_on_start` defaults to `True`. To keep data across runs, use a named dataset or turn that option off. For details, see the [Storages](../concepts/storages) page. To continue an interrupted crawl from where it stopped, see the [Resuming a paused crawl](./stopping-and-resuming-crawlers#resuming-a-paused-crawl) example. ## Concurrency and throttling -Both Scrapy and Crawlee have settings to influence the scraping throughput, but they differ in their approach. Crawlee reads `ConcurrencySettings` and scales concurrency automatically based on system resources, as described in the [Scaling crawlers guide](./scaling-crawlers). Use `max_tasks_per_minute` to cap the request rate. It's close to `DOWNLOAD_DELAY`, but it limits the whole pool rather than adding a fixed delay per domain. +Both Scrapy and Crawlee have settings to influence the scraping throughput, but they differ in their approach. Crawlee reads `ConcurrencySettings` and scales concurrency automatically based on system resources, as described on the [Scaling crawlers](../concepts/scaling-crawlers) page. Use `max_tasks_per_minute` to cap the request rate. It's close to `DOWNLOAD_DELAY`, but it limits the whole pool rather than adding a fixed delay per domain. @@ -210,7 +210,7 @@ Both Scrapy and Crawlee have settings to influence the scraping throughput, but -The closest built-in analog to the `AutoThrottle` extension is the `ThrottlingRequestManager`, covered in the [Request throttling guide](./request-throttling). It differs in what drives the backoff. Scrapy's `AutoThrottle` adapts to measured latency, while Crawlee's manager reacts to explicit signals: HTTP 429 replies and `robots.txt` crawl-delay directives. It wraps a `RequestQueue` and applies the backoff per domain. You list the domains to watch, and the crawler reports the signals to the manager on its own. +The closest built-in analog to the `AutoThrottle` extension is the `ThrottlingRequestManager`, covered on the [Request throttling](../concepts/request-throttling) page. It differs in what drives the backoff. Scrapy's `AutoThrottle` adapts to measured latency, while Crawlee's manager reacts to explicit signals: HTTP 429 replies and `robots.txt` crawl-delay directives. It wraps a `RequestQueue` and applies the backoff per domain. You list the domains to watch, and the crawler reports the signals to the manager on its own. Crawl-delay works only when the crawler fetches `robots.txt`. Projects generated by `scrapy startproject` obey `robots.txt` out of the box, but the default for Crawlee is to ignore it. Passing `respect_robots_txt_file=True` tells Crawlee to enforce the `Disallow` rules. Together with `ThrottlingRequestManager` it also reads crawl-delay, otherwise it logs a warning. @@ -229,7 +229,7 @@ Crawl-delay works only when the crawler fetches `robots.txt`. Projects generated ## Proxies -Scrapy rotates proxies through a middleware or a third-party package. Crawlee takes a `ProxyConfiguration` and rotates the URLs for you. For tiered proxies, session-bound proxies, and integration with the Apify Proxy, see the [Proxy management guide](./proxy-management). +Scrapy rotates proxies through a middleware or a third-party package. Crawlee takes a `ProxyConfiguration` and rotates the URLs for you. For tiered proxies, session-bound proxies, and integration with the Apify Proxy, see the [Proxy management](../concepts/proxy-management) page. @@ -246,7 +246,7 @@ Scrapy rotates proxies through a middleware or a third-party package. Crawlee ta ## Error handling -Scrapy retries failed requests with `RetryMiddleware` and reports terminal failures through `errback`. Crawlee retries with `max_request_retries`, runs an `error_handler` between attempts, and calls a `failed_request_handler` once the retries run out. For details, see the [Error handling guide](./error-handling). +Scrapy retries failed requests with `RetryMiddleware` and reports terminal failures through `errback`. Crawlee retries with `max_request_retries`, runs an `error_handler` between attempts, and calls a `failed_request_handler` once the retries run out. For details, see the [Error handling](../concepts/error-handling) page. @@ -280,7 +280,7 @@ Scrapy submits forms with `FormRequest.from_response`, which reads the form from ## JavaScript rendering -For pages that need a browser, Scrapy users add the `scrapy-playwright` package, which requires extra settings to register its download handlers and the asyncio reactor. Crawlee has browser support built in through the `PlaywrightCrawler`. The handler receives a [Playwright `Page`](https://playwright.dev/python/docs/api/class-page) on `context.page`, so you query the rendered DOM instead of a static response. For details, see the [Playwright crawler guide](./playwright-crawler). +For pages that need a browser, Scrapy users add the `scrapy-playwright` package, which requires extra settings to register its download handlers and the asyncio reactor. Crawlee has browser support built in through the `PlaywrightCrawler`. The handler receives a [Playwright `Page`](https://playwright.dev/python/docs/api/class-page) on `context.page`, so you query the rendered DOM instead of a static response. For details, see the [Playwright crawler](../concepts/playwright-crawler) page. @@ -298,7 +298,7 @@ For pages that need a browser, Scrapy users add the `scrapy-playwright` package, -`PlaywrightCrawler` renders every request. In Scrapy you flag individual requests with `meta={'playwright': True}` and leave the rest on the plain downloader. For that mix in Crawlee, use `AdaptivePlaywrightCrawler`, which renders in a browser only when the page needs it. For details, see the [Adaptive Playwright crawler guide](./adaptive-playwright-crawler). +`PlaywrightCrawler` renders every request. In Scrapy you flag individual requests with `meta={'playwright': True}` and leave the rest on the plain downloader. For that mix in Crawlee, use `AdaptivePlaywrightCrawler`, which renders in a browser only when the page needs it. For details, see the [Adaptive Playwright crawler](../concepts/adaptive-playwright-crawler) page. ## Conclusion diff --git a/docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py b/docs/04_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py similarity index 100% rename from docs/guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py rename to docs/04_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py diff --git a/docs/examples/code_examples/playwright_crawler_with_camoufox.py b/docs/04_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py similarity index 100% rename from docs/examples/code_examples/playwright_crawler_with_camoufox.py rename to docs/04_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py diff --git a/docs/guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py b/docs/04_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py similarity index 100% rename from docs/guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py rename to docs/04_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py diff --git a/docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py b/docs/04_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py similarity index 100% rename from docs/guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py rename to docs/04_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py diff --git a/docs/examples/code_examples/crawl_all_links_on_website_bs.py b/docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py similarity index 100% rename from docs/examples/code_examples/crawl_all_links_on_website_bs.py rename to docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py diff --git a/docs/examples/code_examples/crawl_all_links_on_website_pw.py b/docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py similarity index 100% rename from docs/examples/code_examples/crawl_all_links_on_website_pw.py rename to docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py diff --git a/docs/examples/code_examples/crawl_multiple_urls_bs.py b/docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py similarity index 100% rename from docs/examples/code_examples/crawl_multiple_urls_bs.py rename to docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py diff --git a/docs/examples/code_examples/crawl_multiple_urls_pw.py b/docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py similarity index 100% rename from docs/examples/code_examples/crawl_multiple_urls_pw.py rename to docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_bs.py b/docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py similarity index 100% rename from docs/examples/code_examples/crawl_specific_links_on_website_bs.py rename to docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py diff --git a/docs/examples/code_examples/crawl_specific_links_on_website_pw.py b/docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py similarity index 100% rename from docs/examples/code_examples/crawl_specific_links_on_website_pw.py rename to docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_all_links.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py similarity index 100% rename from docs/examples/code_examples/crawl_website_with_relative_links_all_links.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py similarity index 100% rename from docs/examples/code_examples/crawl_website_with_relative_links_same_domain.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py similarity index 100% rename from docs/examples/code_examples/crawl_website_with_relative_links_same_hostname.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py diff --git a/docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py similarity index 100% rename from docs/examples/code_examples/crawl_website_with_relative_links_same_origin.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py diff --git a/docs/examples/code_examples/extract_and_add_specific_links_on_website_bs.py b/docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py similarity index 100% rename from docs/examples/code_examples/extract_and_add_specific_links_on_website_bs.py rename to docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py diff --git a/docs/examples/code_examples/extract_and_add_specific_links_on_website_pw.py b/docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py similarity index 100% rename from docs/examples/code_examples/extract_and_add_specific_links_on_website_pw.py rename to docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py diff --git a/docs/guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py b/docs/04_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py similarity index 100% rename from docs/guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py rename to docs/04_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py diff --git a/docs/guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py b/docs/04_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py similarity index 100% rename from docs/guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py rename to docs/04_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py diff --git a/docs/guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py b/docs/04_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py similarity index 100% rename from docs/guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py rename to docs/04_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py diff --git a/docs/examples/code_examples/fill_and_submit_web_form_automated.py b/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py similarity index 100% rename from docs/examples/code_examples/fill_and_submit_web_form_automated.py rename to docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py diff --git a/docs/examples/code_examples/fill_and_submit_web_form_crawler.py b/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py similarity index 100% rename from docs/examples/code_examples/fill_and_submit_web_form_crawler.py rename to docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py diff --git a/docs/examples/code_examples/fill_and_submit_web_form_request.py b/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py similarity index 100% rename from docs/examples/code_examples/fill_and_submit_web_form_request.py rename to docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py diff --git a/docs/guides/code_examples/login_crawler/http_login.py b/docs/04_guides/code_examples/login_crawler/http_login.py similarity index 100% rename from docs/guides/code_examples/login_crawler/http_login.py rename to docs/04_guides/code_examples/login_crawler/http_login.py diff --git a/docs/guides/code_examples/login_crawler/playwright_login.py b/docs/04_guides/code_examples/login_crawler/playwright_login.py similarity index 100% rename from docs/guides/code_examples/login_crawler/playwright_login.py rename to docs/04_guides/code_examples/login_crawler/playwright_login.py diff --git a/docs/examples/code_examples/using_browser_profiles_chrome.py b/docs/04_guides/code_examples/login_crawler/using_browser_profiles_chrome.py similarity index 100% rename from docs/examples/code_examples/using_browser_profiles_chrome.py rename to docs/04_guides/code_examples/login_crawler/using_browser_profiles_chrome.py diff --git a/docs/examples/code_examples/using_browser_profiles_firefox.py b/docs/04_guides/code_examples/login_crawler/using_browser_profiles_firefox.py similarity index 100% rename from docs/examples/code_examples/using_browser_profiles_firefox.py rename to docs/04_guides/code_examples/login_crawler/using_browser_profiles_firefox.py diff --git a/docs/guides/code_examples/pydantic_ai_crawler/additional_instructions_example.py b/docs/04_guides/code_examples/pydantic_ai_crawler/additional_instructions_example.py similarity index 100% rename from docs/guides/code_examples/pydantic_ai_crawler/additional_instructions_example.py rename to docs/04_guides/code_examples/pydantic_ai_crawler/additional_instructions_example.py diff --git a/docs/guides/code_examples/pydantic_ai_crawler/basic_example.py b/docs/04_guides/code_examples/pydantic_ai_crawler/basic_example.py similarity index 100% rename from docs/guides/code_examples/pydantic_ai_crawler/basic_example.py rename to docs/04_guides/code_examples/pydantic_ai_crawler/basic_example.py diff --git a/docs/guides/code_examples/pydantic_ai_crawler/custom_distiller_example.py b/docs/04_guides/code_examples/pydantic_ai_crawler/custom_distiller_example.py similarity index 100% rename from docs/guides/code_examples/pydantic_ai_crawler/custom_distiller_example.py rename to docs/04_guides/code_examples/pydantic_ai_crawler/custom_distiller_example.py diff --git a/docs/guides/code_examples/pydantic_ai_crawler/debugging_example.py b/docs/04_guides/code_examples/pydantic_ai_crawler/debugging_example.py similarity index 100% rename from docs/guides/code_examples/pydantic_ai_crawler/debugging_example.py rename to docs/04_guides/code_examples/pydantic_ai_crawler/debugging_example.py diff --git a/docs/guides/code_examples/pydantic_ai_crawler/selector_extractor_example.py b/docs/04_guides/code_examples/pydantic_ai_crawler/selector_extractor_example.py similarity index 100% rename from docs/guides/code_examples/pydantic_ai_crawler/selector_extractor_example.py rename to docs/04_guides/code_examples/pydantic_ai_crawler/selector_extractor_example.py diff --git a/docs/guides/code_examples/pydantic_ai_crawler/usage_limit_example.py b/docs/04_guides/code_examples/pydantic_ai_crawler/usage_limit_example.py similarity index 100% rename from docs/guides/code_examples/pydantic_ai_crawler/usage_limit_example.py rename to docs/04_guides/code_examples/pydantic_ai_crawler/usage_limit_example.py diff --git a/docs/examples/code_examples/respect_robots_on_skipped_request.py b/docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py similarity index 100% rename from docs/examples/code_examples/respect_robots_on_skipped_request.py rename to docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py diff --git a/docs/examples/code_examples/respect_robots_txt_file.py b/docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py similarity index 100% rename from docs/examples/code_examples/respect_robots_txt_file.py rename to docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py diff --git a/docs/examples/code_examples/run_parallel_crawlers.py b/docs/04_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py similarity index 100% rename from docs/examples/code_examples/run_parallel_crawlers.py rename to docs/04_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py diff --git a/docs/introduction/code_examples/__init__.py b/docs/04_guides/code_examples/running_in_web_server/__init__.py similarity index 100% rename from docs/introduction/code_examples/__init__.py rename to docs/04_guides/code_examples/running_in_web_server/__init__.py diff --git a/docs/guides/code_examples/running_in_web_server/crawler.py b/docs/04_guides/code_examples/running_in_web_server/crawler.py similarity index 100% rename from docs/guides/code_examples/running_in_web_server/crawler.py rename to docs/04_guides/code_examples/running_in_web_server/crawler.py diff --git a/docs/guides/code_examples/running_in_web_server/server.py b/docs/04_guides/code_examples/running_in_web_server/server.py similarity index 100% rename from docs/guides/code_examples/running_in_web_server/server.py rename to docs/04_guides/code_examples/running_in_web_server/server.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_concurrency.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_concurrency.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_concurrency.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_concurrency.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_crawlspider.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_crawlspider.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_crawlspider.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_crawlspider.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_error_handling.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_error_handling.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_error_handling.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_error_handling.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_export.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_export.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_export.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_export.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_labels.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_labels.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_labels.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_labels.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_playwright.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_playwright.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_playwright.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_playwright.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_post.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_post.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_post.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_proxy.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_proxy.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_proxy.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_proxy.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_quotes.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_quotes.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_quotes.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_quotes.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_throttling.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_throttling.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_throttling.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_throttling.py diff --git a/docs/guides/code_examples/scrapy_migration/crawlee_user_data.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_user_data.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/crawlee_user_data.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_user_data.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_authors.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_authors.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_authors.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_authors.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_concurrency.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_concurrency.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_concurrency.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_crawlspider.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_crawlspider.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_crawlspider.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_crawlspider.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_errback.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_errback.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_errback.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_errback.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_export.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_export.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_export.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_export.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_formrequest.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_formrequest.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_formrequest.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_formrequest.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_playwright.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_playwright.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_playwright.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_playwright.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_playwright_settings.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_playwright_settings.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_proxy.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_proxy.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_proxy.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_proxy.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_quotes.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_quotes.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_quotes.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_quotes.py diff --git a/docs/guides/code_examples/scrapy_migration/scrapy_throttling.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_throttling.py similarity index 100% rename from docs/guides/code_examples/scrapy_migration/scrapy_throttling.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_throttling.py diff --git a/docs/guides/code_examples/stagehand_crawler/basic_example.py b/docs/04_guides/code_examples/stagehand_crawler/basic_example.py similarity index 100% rename from docs/guides/code_examples/stagehand_crawler/basic_example.py rename to docs/04_guides/code_examples/stagehand_crawler/basic_example.py diff --git a/docs/guides/code_examples/stagehand_crawler/browserbase_example.py b/docs/04_guides/code_examples/stagehand_crawler/browserbase_example.py similarity index 100% rename from docs/guides/code_examples/stagehand_crawler/browserbase_example.py rename to docs/04_guides/code_examples/stagehand_crawler/browserbase_example.py diff --git a/docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py b/docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py similarity index 100% rename from docs/examples/code_examples/beautifulsoup_crawler_keep_alive.py rename to docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py diff --git a/docs/examples/code_examples/beautifulsoup_crawler_stop.py b/docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py similarity index 100% rename from docs/examples/code_examples/beautifulsoup_crawler_stop.py rename to docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py diff --git a/docs/examples/code_examples/resuming_paused_crawl.py b/docs/04_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py similarity index 100% rename from docs/examples/code_examples/resuming_paused_crawl.py rename to docs/04_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py diff --git a/docs/guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py b/docs/04_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py similarity index 100% rename from docs/guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py rename to docs/04_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py diff --git a/docs/deployment/apify_platform.mdx b/docs/05_deployment/apify_platform.mdx similarity index 97% rename from docs/deployment/apify_platform.mdx rename to docs/05_deployment/apify_platform.mdx index 426bcc3d2a..88db05ca7d 100644 --- a/docs/deployment/apify_platform.mdx +++ b/docs/05_deployment/apify_platform.mdx @@ -13,7 +13,7 @@ import CrawlerAsActorExample from '!!raw-loader!./code_examples/apify/crawler_as import ProxyExample from '!!raw-loader!./code_examples/apify/proxy_example.py'; import ProxyAdvancedExample from '!!raw-loader!./code_examples/apify/proxy_advanced_example.py'; -Apify is a [platform](https://apify.com) built to serve large-scale and high-performance web scraping and automation needs. It provides easy access to [compute instances (Actors)](#what-is-an-actor), convenient request and result storages, [proxies](../guides/proxy-management), scheduling, webhooks, and [more in the Apify documentation](https://docs.apify.com/), accessible through a [web interface](https://console.apify.com) or an [API](https://docs.apify.com/api). +Apify is a [platform](https://apify.com) built to serve large-scale and high-performance web scraping and automation needs. It provides easy access to [compute instances (Actors)](#what-is-an-actor), convenient request and result storages, [proxies](../concepts/proxy-management), scheduling, webhooks, and [more in the Apify documentation](https://docs.apify.com/), accessible through a [web interface](https://console.apify.com) or an [API](https://docs.apify.com/api). While we think that the Apify platform is super cool, and it's definitely worth signing up for a [free account](https://console.apify.com/sign-up), **Crawlee is and will always be open source**, runnable locally or on any cloud infrastructure. @@ -127,7 +127,7 @@ Your script will be uploaded to and built on the Apify platform so that it can b ## Usage on Apify platform -You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Examples](../examples) section. +You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Guides](../guides) section. ## Storages diff --git a/docs/deployment/aws_lambda.mdx b/docs/05_deployment/aws_lambda.mdx similarity index 99% rename from docs/deployment/aws_lambda.mdx rename to docs/05_deployment/aws_lambda.mdx index b07d1a2928..8d7d94123a 100644 --- a/docs/deployment/aws_lambda.mdx +++ b/docs/05_deployment/aws_lambda.mdx @@ -14,7 +14,7 @@ import PlaywrightCrawlerDockerfile from '!!raw-loader!./code_examples/aws/playwr [AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) is a serverless compute service that lets you run code without provisioning or managing servers. This guide covers deploying `BeautifulSoupCrawler` and `PlaywrightCrawler`. -The code examples are based on the [BeautifulSoupCrawler example](../examples/beautifulsoup-crawler). +The code examples are based on the [BeautifulSoupCrawler example](../concepts/http-crawlers#beautifulsoupcrawler). ## BeautifulSoupCrawler on AWS Lambda diff --git a/docs/deployment/code_examples/apify/crawler_as_actor_example.py b/docs/05_deployment/code_examples/apify/crawler_as_actor_example.py similarity index 100% rename from docs/deployment/code_examples/apify/crawler_as_actor_example.py rename to docs/05_deployment/code_examples/apify/crawler_as_actor_example.py diff --git a/docs/deployment/code_examples/apify/get_public_url.py b/docs/05_deployment/code_examples/apify/get_public_url.py similarity index 100% rename from docs/deployment/code_examples/apify/get_public_url.py rename to docs/05_deployment/code_examples/apify/get_public_url.py diff --git a/docs/deployment/code_examples/apify/log_with_config_example.py b/docs/05_deployment/code_examples/apify/log_with_config_example.py similarity index 100% rename from docs/deployment/code_examples/apify/log_with_config_example.py rename to docs/05_deployment/code_examples/apify/log_with_config_example.py diff --git a/docs/deployment/code_examples/apify/proxy_advanced_example.py b/docs/05_deployment/code_examples/apify/proxy_advanced_example.py similarity index 100% rename from docs/deployment/code_examples/apify/proxy_advanced_example.py rename to docs/05_deployment/code_examples/apify/proxy_advanced_example.py diff --git a/docs/deployment/code_examples/apify/proxy_example.py b/docs/05_deployment/code_examples/apify/proxy_example.py similarity index 100% rename from docs/deployment/code_examples/apify/proxy_example.py rename to docs/05_deployment/code_examples/apify/proxy_example.py diff --git a/docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py b/docs/05_deployment/code_examples/aws/beautifulsoup_crawler_lambda.py similarity index 100% rename from docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py rename to docs/05_deployment/code_examples/aws/beautifulsoup_crawler_lambda.py diff --git a/docs/deployment/code_examples/aws/playwright_crawler_lambda.py b/docs/05_deployment/code_examples/aws/playwright_crawler_lambda.py similarity index 100% rename from docs/deployment/code_examples/aws/playwright_crawler_lambda.py rename to docs/05_deployment/code_examples/aws/playwright_crawler_lambda.py diff --git a/docs/deployment/code_examples/aws/playwright_dockerfile b/docs/05_deployment/code_examples/aws/playwright_dockerfile similarity index 100% rename from docs/deployment/code_examples/aws/playwright_dockerfile rename to docs/05_deployment/code_examples/aws/playwright_dockerfile diff --git a/docs/deployment/code_examples/google/cloud_run_example.py b/docs/05_deployment/code_examples/google/cloud_run_example.py similarity index 100% rename from docs/deployment/code_examples/google/cloud_run_example.py rename to docs/05_deployment/code_examples/google/cloud_run_example.py diff --git a/docs/deployment/code_examples/google/google_example.py b/docs/05_deployment/code_examples/google/google_example.py similarity index 100% rename from docs/deployment/code_examples/google/google_example.py rename to docs/05_deployment/code_examples/google/google_example.py diff --git a/docs/deployment/google_cloud.mdx b/docs/05_deployment/google_cloud.mdx similarity index 97% rename from docs/deployment/google_cloud.mdx rename to docs/05_deployment/google_cloud.mdx index e4f1fbe480..5c4a3535cf 100644 --- a/docs/deployment/google_cloud.mdx +++ b/docs/05_deployment/google_cloud.mdx @@ -14,7 +14,7 @@ import GoogleFunctions from '!!raw-loader!./code_examples/google/google_example. ## Updating the project -For the project foundation, use BeautifulSoupCrawler as described in this [example](../examples/beautifulsoup-crawler). +For the project foundation, use BeautifulSoupCrawler as described in this [example](../concepts/http-crawlers#beautifulsoupcrawler). Add [`functions-framework`](https://pypi.org/project/functions-framework/) to your dependencies file `requirements.txt`. If you're using a project manager like `poetry` or `uv`, export your dependencies to `requirements.txt`. diff --git a/docs/deployment/google_cloud_run.mdx b/docs/05_deployment/google_cloud_run.mdx similarity index 100% rename from docs/deployment/google_cloud_run.mdx rename to docs/05_deployment/google_cloud_run.mdx diff --git a/docs/upgrading/upgrading_to_v0x.md b/docs/06_upgrading/upgrading_to_v0x.md similarity index 98% rename from docs/upgrading/upgrading_to_v0x.md rename to docs/06_upgrading/upgrading_to_v0x.md index 0ce52466b2..99b657368c 100644 --- a/docs/upgrading/upgrading_to_v0x.md +++ b/docs/06_upgrading/upgrading_to_v0x.md @@ -1,6 +1,7 @@ --- id: upgrading-to-v0x title: Upgrading to v0.x +description: Breaking changes and migration steps for upgrading between Crawlee for Python v0.x versions. --- This page summarizes the breaking changes between Crawlee for Python zero-based versions. diff --git a/docs/upgrading/upgrading_to_v1.md b/docs/06_upgrading/upgrading_to_v1.md similarity index 97% rename from docs/upgrading/upgrading_to_v1.md rename to docs/06_upgrading/upgrading_to_v1.md index 412a8919f2..ba2f37e58b 100644 --- a/docs/upgrading/upgrading_to_v1.md +++ b/docs/06_upgrading/upgrading_to_v1.md @@ -1,6 +1,7 @@ --- id: upgrading-to-v1 title: Upgrading to v1 +description: Breaking changes and migration guide for upgrading from Crawlee for Python v0.x to v1.0. --- This page summarizes the breaking changes between Crawlee for Python v0.6 and v1.0. @@ -52,13 +53,13 @@ client = HttpxHttpClient() crawler = HttpCrawler(http_client=client) ``` -See the [HTTP clients guide](https://crawlee.dev/python/docs/guides/http-clients) for all options. +See the [HTTP clients](../concepts/http-clients) page for all options. ## Changes in storages In Crawlee v1.0, the `Dataset`, `KeyValueStore`, and `RequestQueue` storage APIs have been updated for consistency and simplicity. Below is a detailed overview of what's new, what's changed, and what's been removed. -See the [Storages guide](https://crawlee.dev/python/docs/guides/storages) for more details. +See the [Storages](../concepts/storages) page for more details. ### Dataset @@ -115,7 +116,7 @@ Some changes in the related model classes: In v1.0, the storage client system has been completely reworked to simplify implementation and make custom storage clients easier to write. -See the [Storage clients guide](https://crawlee.dev/python/docs/guides/storage-clients) for more details. +See the [Storage clients](../concepts/storage-clients) page for more details. ### New dedicated storage clients diff --git a/docs/examples/add_data_to_dataset.mdx b/docs/examples/add_data_to_dataset.mdx deleted file mode 100644 index 697b157a1e..0000000000 --- a/docs/examples/add_data_to_dataset.mdx +++ /dev/null @@ -1,40 +0,0 @@ ---- -id: add-data-to-dataset -title: Add data to dataset ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_bs.py'; -import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_pw.py'; -import DatasetExample from '!!raw-loader!roa-loader!./code_examples/add_data_to_dataset_dataset.py'; - -This example demonstrates how to store extracted data into datasets using the `context.push_data` helper function. If the specified dataset does not already exist, it will be created automatically. Additionally, you can save data to custom datasets by providing `dataset_id` or `dataset_name` parameters to the `push_data` function. - - - - - {BeautifulSoupExample} - - - - - {PlaywrightExample} - - - - -Each item in the dataset will be stored in its own file within the following directory: - -```text -{PROJECT_FOLDER}/storage/datasets/default/ -``` - -For more control, you can also open a dataset manually using the asynchronous constructor `Dataset.open` - - - {DatasetExample} - diff --git a/docs/examples/beautifulsoup_crawler.mdx b/docs/examples/beautifulsoup_crawler.mdx deleted file mode 100644 index 160e4c4d65..0000000000 --- a/docs/examples/beautifulsoup_crawler.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -id: beautifulsoup-crawler -title: BeautifulSoup crawler ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler.py'; - -This example demonstrates how to use `BeautifulSoupCrawler` to crawl a list of URLs, load each URL using a plain HTTP request, parse the HTML using the [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library and extract some data from it - the page title and all `

`, `

` and `

` tags. This setup is perfect for scraping specific elements from web pages. Thanks to the well-known BeautifulSoup, you can easily navigate the HTML structure and retrieve the data you need with minimal code. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request. - - - {BeautifulSoupExample} - diff --git a/docs/examples/capture_screenshot_using_playwright.mdx b/docs/examples/capture_screenshot_using_playwright.mdx deleted file mode 100644 index 614693b1e8..0000000000 --- a/docs/examples/capture_screenshot_using_playwright.mdx +++ /dev/null @@ -1,19 +0,0 @@ ---- -id: capture-screenshots-using-playwright -title: Capture screenshots using Playwright ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import CaptureScreenshotExample from '!!raw-loader!roa-loader!./code_examples/capture_screenshot_using_playwright.py'; - -This example demonstrates how to capture screenshots of web pages using `PlaywrightCrawler` and store them in the key-value store. - -The `PlaywrightCrawler` is configured to automate the browsing and interaction with web pages. It uses headless Chromium as the browser type to perform these tasks. Each web page specified in the initial list of URLs is visited sequentially, and a screenshot of the page is captured using Playwright's `page.screenshot()` method. - -The captured screenshots are stored in the key-value store, which is suitable for managing and storing files in various formats. In this case, screenshots are stored as PNG images with a unique key generated from the URL of the page. - - - {CaptureScreenshotExample} - diff --git a/docs/examples/capturing_page_snapshots_with_error_snapshotter.mdx b/docs/examples/capturing_page_snapshots_with_error_snapshotter.mdx deleted file mode 100644 index 87ff540298..0000000000 --- a/docs/examples/capturing_page_snapshots_with_error_snapshotter.mdx +++ /dev/null @@ -1,27 +0,0 @@ ---- -id: capturing-page-snapshots-with-error-snapshotter -title: Capturing page snapshots with ErrorSnapshotter -description: How to capture page snapshots on errors. ---- -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import ApiLink from '@site/src/components/ApiLink'; -import ParselCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_with_error_snapshotter.py'; -import PlaywrightCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_error_snapshotter.py'; - - -This example demonstrates how to capture page snapshots on first occurrence of each unique error. The capturing happens automatically if you set `save_error_snapshots=True` in the crawler's `Statistics`. The error snapshot can contain `html` file and `jpeg` file that are created from the page where the unhandled exception was raised. Captured error snapshot files are saved to the default key-value store. Both `PlaywrightCrawler` and [HTTP crawlers](../guides/http-crawlers) are capable of capturing the html file, but only `PlaywrightCrawler` is able to capture page screenshot as well. - - - - - { ParselCrawlerWithErrorSnapshotter } - - - - - { PlaywrightCrawlerWithErrorSnapshotter } - - - diff --git a/docs/examples/code_examples/add_data_to_dataset_bs.py b/docs/examples/code_examples/add_data_to_dataset_bs.py deleted file mode 100644 index 4318cbe0d4..0000000000 --- a/docs/examples/code_examples/add_data_to_dataset_bs.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio - -from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext - - -async def main() -> None: - crawler = BeautifulSoupCrawler() - - # Define the default request handler, which will be called for every request. - @crawler.router.default_handler - async def request_handler(context: BeautifulSoupCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Extract data from the page. - data = { - 'url': context.request.url, - 'title': context.soup.title.string if context.soup.title else None, - 'html': str(context.soup)[:1000], - } - - # Push the extracted data to the default dataset. - await context.push_data(data) - - # Run the crawler with the initial list of requests. - await crawler.run( - [ - 'https://crawlee.dev', - 'https://apify.com', - 'https://example.com', - ] - ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_dataset.py b/docs/examples/code_examples/add_data_to_dataset_dataset.py deleted file mode 100644 index b1d9aba923..0000000000 --- a/docs/examples/code_examples/add_data_to_dataset_dataset.py +++ /dev/null @@ -1,15 +0,0 @@ -import asyncio - -from crawlee.storages import Dataset - - -async def main() -> None: - # Open dataset manually using asynchronous constructor open(). - dataset = await Dataset.open() - - # Interact with dataset directly. - await dataset.push_data({'key': 'value'}) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/add_data_to_dataset_pw.py b/docs/examples/code_examples/add_data_to_dataset_pw.py deleted file mode 100644 index 8eb714aef3..0000000000 --- a/docs/examples/code_examples/add_data_to_dataset_pw.py +++ /dev/null @@ -1,35 +0,0 @@ -import asyncio - -from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext - - -async def main() -> None: - crawler = PlaywrightCrawler() - - # Define the default request handler, which will be called for every request. - @crawler.router.default_handler - async def request_handler(context: PlaywrightCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Extract data from the page. - data = { - 'url': context.request.url, - 'title': await context.page.title(), - 'html': str(await context.page.content())[:1000], - } - - # Push the extracted data to the default dataset. - await context.push_data(data) - - # Run the crawler with the initial list of requests. - await crawler.run( - [ - 'https://crawlee.dev', - 'https://apify.com', - 'https://example.com', - ] - ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/beautifulsoup_crawler.py b/docs/examples/code_examples/beautifulsoup_crawler.py deleted file mode 100644 index 5e9701d7cb..0000000000 --- a/docs/examples/code_examples/beautifulsoup_crawler.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio -from datetime import timedelta - -from crawlee.crawlers import ( - BasicCrawlingContext, - BeautifulSoupCrawler, - BeautifulSoupCrawlingContext, -) - - -async def main() -> None: - # Create an instance of the BeautifulSoupCrawler class, a crawler that automatically - # loads the URLs and parses their HTML using the BeautifulSoup library. - crawler = BeautifulSoupCrawler( - # On error, retry each page at most once. - max_request_retries=1, - # Increase the timeout for processing each page to 30 seconds. - request_handler_timeout=timedelta(seconds=30), - # Limit the crawl to max requests. Remove or increase it for crawling all links. - max_requests_per_crawl=10, - ) - - # Define the default request handler, which will be called for every request. - # The handler receives a context parameter, providing various properties and - # helper methods. Here are a few key ones we use for demonstration: - # - request: an instance of the Request class containing details such as the URL - # being crawled and the HTTP method used. - # - soup: the BeautifulSoup object containing the parsed HTML of the response. - @crawler.router.default_handler - async def request_handler(context: BeautifulSoupCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Extract data from the page. - data = { - 'url': context.request.url, - 'title': context.soup.title.string if context.soup.title else None, - 'h1s': [h1.text for h1 in context.soup.find_all('h1')], - 'h2s': [h2.text for h2 in context.soup.find_all('h2')], - 'h3s': [h3.text for h3 in context.soup.find_all('h3')], - } - - # Push the extracted data to the default dataset. In local configuration, - # the data will be stored as JSON files in ./storage/datasets/default. - await context.push_data(data) - - # Register pre navigation hook which will be called before each request. - # This hook is optional and does not need to be defined at all. - @crawler.pre_navigation_hook - async def some_hook(context: BasicCrawlingContext) -> None: - pass - - # Run the crawler with the initial list of URLs. - await crawler.run(['https://crawlee.dev']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/parsel_crawler.py b/docs/examples/code_examples/parsel_crawler.py deleted file mode 100644 index 9807d7ca3b..0000000000 --- a/docs/examples/code_examples/parsel_crawler.py +++ /dev/null @@ -1,47 +0,0 @@ -import asyncio - -from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext - -# Regex for identifying email addresses on a webpage. -EMAIL_REGEX = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' - - -async def main() -> None: - crawler = ParselCrawler( - # Limit the crawl to max requests. Remove or increase it for crawling all links. - max_requests_per_crawl=10, - ) - - # Define the default request handler, which will be called for every request. - @crawler.router.default_handler - async def request_handler(context: ParselCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Extract data from the page. - data = { - 'url': context.request.url, - 'title': context.selector.xpath('//title/text()').get(), - 'email_address_list': context.selector.re(EMAIL_REGEX), - } - - # Push the extracted data to the default dataset. - await context.push_data(data) - - # Enqueue all links found on the page. - await context.enqueue_links() - - # Register pre navigation hook which will be called before each request. - # This hook is optional and does not need to be defined at all. - @crawler.pre_navigation_hook - async def some_hook(context: BasicCrawlingContext) -> None: - pass - - # Run the crawler with the initial list of URLs. - await crawler.run(['https://github.com']) - - # Export the entire dataset to a JSON file. - await crawler.export_data(path='results.json') - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py b/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py deleted file mode 100644 index 24cb5bb907..0000000000 --- a/docs/examples/code_examples/playwright_crawler_with_fingerprint_generator.py +++ /dev/null @@ -1,44 +0,0 @@ -import asyncio - -from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext -from crawlee.fingerprint_suite import ( - DefaultFingerprintGenerator, - HeaderGeneratorOptions, - ScreenOptions, -) - - -async def main() -> None: - # Use default fingerprint generator with desired fingerprint options. - # Generator will generate real looking browser fingerprint based on the options. - # Unspecified fingerprint options will be automatically selected by the generator. - fingerprint_generator = DefaultFingerprintGenerator( - header_options=HeaderGeneratorOptions(browsers=['chrome']), - screen_options=ScreenOptions(min_width=400), - ) - - crawler = PlaywrightCrawler( - # Limit the crawl to max requests. Remove or increase it for crawling all links. - max_requests_per_crawl=10, - # Headless mode, set to False to see the browser in action. - headless=False, - # Browser types supported by Playwright. - browser_type='chromium', - # Fingerprint generator to be used. By default no fingerprint generation is done. - fingerprint_generator=fingerprint_generator, - ) - - # Define the default request handler, which will be called for every request. - @crawler.router.default_handler - async def request_handler(context: PlaywrightCrawlingContext) -> None: - context.log.info(f'Processing {context.request.url} ...') - - # Find a link to the next page and enqueue it if it exists. - await context.enqueue_links(selector='.morelink') - - # Run the crawler with the initial list of URLs. - await crawler.run(['https://news.ycombinator.com/']) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/examples/crawl_all_links_on_website.mdx b/docs/examples/crawl_all_links_on_website.mdx deleted file mode 100644 index f17c63920f..0000000000 --- a/docs/examples/crawl_all_links_on_website.mdx +++ /dev/null @@ -1,33 +0,0 @@ ---- -id: crawl-all-links-on-website -title: Crawl all links on website ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_bs.py'; -import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_all_links_on_website_pw.py'; - -This example uses the `enqueue_links` helper to add new links to the `RequestQueue` as the crawler navigates from page to page. By automatically discovering and enqueuing all links on a given page, the crawler can systematically scrape an entire website. This approach is ideal for web scraping tasks where you need to collect data from multiple interconnected pages. - -:::tip - -If no options are given, by default the method will only add links that are under the same subdomain. This behavior can be controlled with the `strategy` option, which is an instance of the `EnqueueStrategy` type alias. You can find more info about this option in the [Crawl website with relative links](./crawl-website-with-relative-links) example. - -::: - - - - - {BeautifulSoupExample} - - - - - {PlaywrightExample} - - - diff --git a/docs/examples/crawl_multiple_urls.mdx b/docs/examples/crawl_multiple_urls.mdx deleted file mode 100644 index 2d3d370283..0000000000 --- a/docs/examples/crawl_multiple_urls.mdx +++ /dev/null @@ -1,27 +0,0 @@ ---- -id: crawl-multiple-urls -title: Crawl multiple URLs ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_bs.py'; -import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_multiple_urls_pw.py'; - -This example demonstrates how to crawl a specified list of URLs using different crawlers. You'll learn how to set up the crawler, define a request handler, and run the crawler with multiple URLs. This setup is useful for scraping data from multiple pages or websites concurrently. - - - - - {BeautifulSoupExample} - - - - - {PlaywrightExample} - - - diff --git a/docs/examples/crawl_specific_links_on_website.mdx b/docs/examples/crawl_specific_links_on_website.mdx deleted file mode 100644 index b350568421..0000000000 --- a/docs/examples/crawl_specific_links_on_website.mdx +++ /dev/null @@ -1,47 +0,0 @@ ---- -id: crawl-specific-links-on-website -title: Crawl specific links on website ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_bs.py'; -import PlaywrightExample from '!!raw-loader!roa-loader!./code_examples/crawl_specific_links_on_website_pw.py'; - -import BeautifulSoupExampleExtractAndAdd from '!!raw-loader!roa-loader!./code_examples/extract_and_add_specific_links_on_website_bs.py'; -import PlaywrightExampleExtractAndAdd from '!!raw-loader!roa-loader!./code_examples/extract_and_add_specific_links_on_website_pw.py'; - -This example demonstrates how to crawl a website while targeting specific patterns of links. By utilizing the `enqueue_links` helper, you can pass `include` or `exclude` parameters to improve your crawling strategy. This approach ensures that only the links matching the specified patterns are added to the `RequestQueue`. Both `include` and `exclude` support lists of globs or regular expressions. This functionality is great for focusing on relevant sections of a website and avoiding scraping unnecessary or irrelevant content. - - - - - {BeautifulSoupExample} - - - - - {PlaywrightExample} - - - - -## Even more control over the enqueued links - -`enqueue_links` is a convenience helper and internally it calls `extract_links` to find the links and `add_requests` to add them to the queue. If you need some additional custom filtering of the extracted links before enqueuing them, then consider using `extract_links` and `add_requests` instead of the `enqueue_links` - - - - - {BeautifulSoupExampleExtractAndAdd} - - - - - {PlaywrightExampleExtractAndAdd} - - - diff --git a/docs/examples/crawl_website_with_relative_links.mdx b/docs/examples/crawl_website_with_relative_links.mdx deleted file mode 100644 index 4cf7bee845..0000000000 --- a/docs/examples/crawl_website_with_relative_links.mdx +++ /dev/null @@ -1,52 +0,0 @@ ---- -id: crawl-website-with-relative-links -title: Crawl website with relative links ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import AllLinksExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_all_links.py'; -import SameDomainExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_domain.py'; -import SameHostnameExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_hostname.py'; -import SameOriginExample from '!!raw-loader!roa-loader!./code_examples/crawl_website_with_relative_links_same_origin.py'; - -When crawling a website, you may encounter various types of links that you wish to include in your crawl. To facilitate this, we provide the `enqueue_links` method on the crawler context, which will automatically find and add these links to the crawler's `RequestQueue`. This method simplifies the process of handling different types of links, including relative links, by automatically resolving them based on the page's context. - -:::note - -For these examples, we are using the `BeautifulSoupCrawler`. However, the same method is available for other crawlers as well. You can use it in exactly the same way. - -::: - -`EnqueueStrategy` type alias provides four distinct strategies for crawling relative links: - -- `all` - Enqueues all links found, regardless of the domain they point to. This strategy is useful when you want to follow every link, including those that navigate to external websites. -- `same-domain` - Enqueues all links found that share the same domain name, including any possible subdomains. This strategy ensures that all links within the same top-level and base domain are included. -- `same-hostname` - Enqueues all links found for the exact same hostname. This is the **default** strategy, and it restricts the crawl to links that have the same hostname as the current page, excluding subdomains. -- `same-origin` - Enqueues all links found that share the same origin. The same origin refers to URLs that share the same protocol, domain, and port, ensuring a strict scope for the crawl. - - - - - {AllLinksExample} - - - - - {SameDomainExample} - - - - - {SameHostnameExample} - - - - - {SameOriginExample} - - - diff --git a/docs/examples/crawler_keep_alive.mdx b/docs/examples/crawler_keep_alive.mdx deleted file mode 100644 index 2e6c6640c7..0000000000 --- a/docs/examples/crawler_keep_alive.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -id: crawler-keep-alive -title: Keep a Crawler alive waiting for more requests ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_keep_alive.py'; - -This example demonstrates how to keep crawler alive even when there are no requests at the moment by using `keep_alive=True` argument of `BasicCrawler.__init__`. This is available to all crawlers that inherit from `BasicCrawler` and in the example below it is shown on `BeautifulSoupCrawler`. To stop the crawler that was started with `keep_alive=True` you can call `crawler.stop()`. - - - {BeautifulSoupExample} - diff --git a/docs/examples/crawler_stop.mdx b/docs/examples/crawler_stop.mdx deleted file mode 100644 index 4ea7f28565..0000000000 --- a/docs/examples/crawler_stop.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -id: crawler-stop -title: Stopping a Crawler with stop method ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import BeautifulSoupExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_stop.py'; - -This example demonstrates how to use `stop` method of `BasicCrawler` to stop crawler once the crawler finds what it is looking for. This method is available to all crawlers that inherit from `BasicCrawler` and in the example below it is shown on `BeautifulSoupCrawler`. Simply call `crawler.stop()` to stop the crawler. It will not continue to crawl through new requests. Requests that are already being concurrently processed are going to get finished. It is possible to call `stop` method with optional argument `reason` that is a string that will be used in logs and it can improve logs readability especially if you have multiple different conditions for triggering `stop`. - - - {BeautifulSoupExample} - diff --git a/docs/examples/export_entire_dataset_to_file.mdx b/docs/examples/export_entire_dataset_to_file.mdx deleted file mode 100644 index 17c2a10a68..0000000000 --- a/docs/examples/export_entire_dataset_to_file.mdx +++ /dev/null @@ -1,47 +0,0 @@ ---- -id: export-entire-dataset-to-file -title: Export entire dataset to file ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import JsonExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_json.py'; -import CsvExample from '!!raw-loader!roa-loader!./code_examples/export_entire_dataset_to_file_csv.py'; - -This example demonstrates how to use the `BasicCrawler.export_data` method of the crawler to export the entire default dataset to a single file. This method supports exporting data in either CSV or JSON format and also accepts additional keyword arguments so you can fine-tune the underlying `json.dump` or `csv.DictWriter` behavior. - -:::note - -For these examples, we are using the `BeautifulSoupCrawler`. However, the same method is available for other crawlers as well. You can use it in exactly the same way. - -::: - - - - - {JsonExample} - - - - - {CsvExample} - - - - -## CSV columns - -Dataset items don't have to share a schema. Different handlers can push different fields, so the items of one dataset often have different keys. - -By default, the CSV columns are the keys of the first non-empty item. When a later item has a key the first one doesn't, its value isn't written, and Crawlee logs a warning naming the dropped keys. To use the keys of all items as columns instead, pass `collect_all_keys=True`: - -```python -await crawler.export_data(path='results.csv', collect_all_keys=True) -``` - -No value is dropped in that mode. Collecting all keys means reading the whole dataset before the first row can be written, so only the default mode writes rows as it goes. - -In both modes, cells for columns an item doesn't have stay empty, or hold the `restval` value if you pass one. diff --git a/docs/examples/file_download.mdx b/docs/examples/file_download.mdx deleted file mode 100644 index 40c859cbe2..0000000000 --- a/docs/examples/file_download.mdx +++ /dev/null @@ -1,25 +0,0 @@ ---- -id: file-download -title: Download files ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import CodeBlock from '@theme/CodeBlock'; - -import FileDownloadExample from '!!raw-loader!roa-loader!./code_examples/file_download.py'; -import FileDownloadStreamExample from '!!raw-loader!./code_examples/file_download_stream.py'; - -This example demonstrates how to use `FileDownloadCrawler` to download files such as PDFs, images or videos with plain HTTP requests. The crawler doesn't parse the response body, so any content type is accepted. Each downloaded file is saved to the default `KeyValueStore` together with the content type reported by the server. - - - {FileDownloadExample} - - -## Streaming large files - -Buffering a whole file in memory doesn't scale to large downloads. Construct the crawler with `stream=True` and the request handler receives a response whose body hasn't been read yet. Consume it in chunks with `read_stream()` and write each chunk to disk as it arrives. - - - {FileDownloadStreamExample} - diff --git a/docs/examples/json_logging.mdx b/docs/examples/json_logging.mdx deleted file mode 100644 index 06dd2ac492..0000000000 --- a/docs/examples/json_logging.mdx +++ /dev/null @@ -1,57 +0,0 @@ ---- -id: configure-json-logging -title: ะกonfigure JSON logging ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import JsonLoggingExample from '!!raw-loader!roa-loader!./code_examples/configure_json_logging.py'; - -This example demonstrates how to configure JSON line (JSONL) logging with Crawlee. By using the `use_table_logs=False` parameter, you can disable table-formatted statistics logs, which makes it easier to parse logs with external tools or to serialize them as JSON. - -The example shows how to integrate with the popular [`loguru`](https://github.com/delgan/loguru) library to capture Crawlee logs and format them as JSONL (one JSON object per line). This approach works well when you need to collect logs for analysis, monitoring, or when integrating with logging platforms like ELK Stack, Grafana Loki, or similar systems. - - - {JsonLoggingExample} - - -Here's an example of what a crawler statistics log entry in JSONL format. - -```json -{ - "text": "[HttpCrawler] | INFO | - Final request statistics: {'requests_finished': 1, 'requests_failed': 0, 'retry_histogram': [1], 'request_avg_failed_duration': None, 'request_avg_finished_duration': 3.57098, 'requests_finished_per_minute': 17, 'requests_failed_per_minute': 0, 'request_total_duration': 3.57098, 'requests_total': 1, 'crawler_runtime': 3.59165}\n", - "record": { - "elapsed": { "repr": "0:00:05.604568", "seconds": 5.604568 }, - "exception": null, - "extra": { - "requests_finished": 1, - "requests_failed": 0, - "retry_histogram": [1], - "request_avg_failed_duration": null, - "request_avg_finished_duration": 3.57098, - "requests_finished_per_minute": 17, - "requests_failed_per_minute": 0, - "request_total_duration": 3.57098, - "requests_total": 1, - "crawler_runtime": 3.59165 - }, - "file": { - "name": "_basic_crawler.py", - "path": "/crawlers/_basic/_basic_crawler.py" - }, - "function": "run", - "level": { "icon": "โ„น๏ธ", "name": "INFO", "no": 20 }, - "line": 583, - "message": "Final request statistics:", - "module": "_basic_crawler", - "name": "HttpCrawler", - "process": { "id": 198383, "name": "MainProcess" }, - "thread": { "id": 135312814966592, "name": "MainThread" }, - "time": { - "repr": "2025-03-17 17:14:45.339150+00:00", - "timestamp": 1742231685.33915 - } - } -} -``` diff --git a/docs/examples/parsel_crawler.mdx b/docs/examples/parsel_crawler.mdx deleted file mode 100644 index b0eca7eb28..0000000000 --- a/docs/examples/parsel_crawler.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -id: parsel-crawler -title: Parsel crawler ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler.py'; - -This example shows how to use `ParselCrawler` to crawl a website or a list of URLs. Each URL is loaded using a plain HTTP request and the response is parsed using [Parsel](https://pypi.org/project/parsel/) library which supports CSS and XPath selectors for HTML responses and JMESPath for JSON responses. We can extract data from all kinds of complex HTML structures using XPath. In this example, we will use Parsel to crawl github.com and extract page title, URL and emails found in the webpage. The default handler will scrape data from the current webpage and enqueue all the links found in the webpage for continuous scraping. It also shows how you can add optional pre-navigation hook to the crawler. Pre-navigation hooks are user defined functions that execute before sending the request. - - - {ParselCrawlerExample} - diff --git a/docs/examples/playwright_crawler.mdx b/docs/examples/playwright_crawler.mdx deleted file mode 100644 index 70b0bc8afb..0000000000 --- a/docs/examples/playwright_crawler.mdx +++ /dev/null @@ -1,19 +0,0 @@ ---- -id: playwright-crawler -title: Playwright crawler ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler.py'; - -This example demonstrates how to use `PlaywrightCrawler` to recursively scrape the Hacker news website using headless Chromium and Playwright. - -The `PlaywrightCrawler` manages the browser and page instances, simplifying the process of interacting with web pages. In the request handler, Playwright's API is used to extract data from each post on the page. Specifically, it retrieves the title, rank, and URL of each post. Additionally, the handler enqueues links to the next pages to ensure continuous scraping. This setup is ideal for scraping dynamic web pages where JavaScript execution is required to render the content. - -A **pre-navigation hook** can be used to perform actions before navigating to the URL. This hook provides further flexibility in controlling environment and preparing for navigation. - - - {PlaywrightCrawlerExample} - diff --git a/docs/examples/playwright_crawler_adaptive.mdx b/docs/examples/playwright_crawler_adaptive.mdx deleted file mode 100644 index f915f0246f..0000000000 --- a/docs/examples/playwright_crawler_adaptive.mdx +++ /dev/null @@ -1,20 +0,0 @@ ---- -id: adaptive-playwright-crawler -title: Adaptive Playwright crawler ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import AdaptivePlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/adaptive_playwright_crawler.py'; - -This example demonstrates how to use `AdaptivePlaywrightCrawler`. An `AdaptivePlaywrightCrawler` is a combination of `PlaywrightCrawler` and some implementation of HTTP-based crawler such as `ParselCrawler` or `BeautifulSoupCrawler`. -It uses a more limited crawling context interface so that it is able to switch to HTTP-only crawling when it detects that it may bring a performance benefit. - -A [pre-navigation hook](/python/docs/guides/adaptive-playwright-crawler#page-configuration-with-pre-navigation-hooks) can be used to perform actions before navigating to the URL. This hook provides further flexibility in controlling environment and preparing for navigation. Hooks will be executed both for the pages crawled by HTTP-bases sub crawler and playwright based sub crawler. Use `playwright_only=True` to mark hooks that should be executed only for playwright sub crawler. - -For more detailed description please see [Adaptive Playwright crawler guide](/python/docs/guides/adaptive-playwright-crawler) - - - {AdaptivePlaywrightCrawlerExample} - diff --git a/docs/examples/playwright_crawler_with_block_requests.mdx b/docs/examples/playwright_crawler_with_block_requests.mdx deleted file mode 100644 index d7d5e15928..0000000000 --- a/docs/examples/playwright_crawler_with_block_requests.mdx +++ /dev/null @@ -1,27 +0,0 @@ ---- -id: playwright-crawler-with-block-requests -title: Playwright crawler with block requests ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import PlaywrightBlockRequests from '!!raw-loader!roa-loader!./code_examples/playwright_block_requests.py'; - -This example demonstrates how to optimize your `PlaywrightCrawler` performance by blocking unnecessary network requests. - -The primary use case is when you need to scrape or interact with web pages without loading non-essential resources like images, styles, or analytics scripts. This can significantly reduce bandwidth usage and improve crawling speed. - -The `block_requests` helper provides the most efficient way to block requests as it operates directly in the browser. - -By default, `block_requests` will block all URLs including the following patterns: - -```python -['.css', '.webp', '.jpg', '.jpeg', '.png', '.svg', '.gif', '.woff', '.pdf', '.zip'] -``` - -You can also replace the default patterns list with your own by providing `url_patterns`, or extend it by passing additional patterns in `extra_url_patterns`. - - - {PlaywrightBlockRequests} - diff --git a/docs/examples/playwright_crawler_with_camoufox.mdx b/docs/examples/playwright_crawler_with_camoufox.mdx deleted file mode 100644 index b627c9ba34..0000000000 --- a/docs/examples/playwright_crawler_with_camoufox.mdx +++ /dev/null @@ -1,26 +0,0 @@ ---- -id: playwright-crawler-with-camoufox -title: Playwright crawler with Camoufox ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import PlaywrightCrawlerExampleWithCamoufox from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_camoufox.py'; - -This example demonstrates how to integrate Camoufox into `PlaywrightCrawler` using `BrowserPool` with custom `PlaywrightBrowserPlugin`. - -Camoufox is a stealthy minimalistic build of Firefox. For details please visit its homepage https://camoufox.com/ . -To be able to run this example you will need to install camoufox, as it is external tool, and it is not part of the crawlee. For installation please see https://pypi.org/project/camoufox/. - -**Warning!** Camoufox is using custom build of firefox. This build can be hundreds of MB large. -You can either pre-download this file using following command `python3 -m camoufox fetch` or camoufox will download it automatically once you try to run it, and it does not find existing binary. -For more details please refer to: https://github.com/daijro/camoufox/tree/main/pythonlib#camoufox-python-interface - -**Project template -** It is possible to generate project with Python code which includes Camoufox integration into crawlee through crawlee cli. Call `crawlee create` and pick `Playwright-camoufox` when asked for Crawler type. - -The example code after PlayWrightCrawler instantiation is similar to example describing the use of Playwright Crawler. The main difference is that in this example Camoufox will be used as the browser through BrowserPool. - - - {PlaywrightCrawlerExampleWithCamoufox} - diff --git a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx b/docs/examples/playwright_crawler_with_fingerprint_generator.mdx deleted file mode 100644 index 04727cd74c..0000000000 --- a/docs/examples/playwright_crawler_with_fingerprint_generator.mdx +++ /dev/null @@ -1,17 +0,0 @@ ---- -id: playwright-crawler-with-fingerprint-generator -title: Playwright crawler with fingerprint generator ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_with_fingerprint_generator.py'; - -This example demonstrates how to use `PlaywrightCrawler` together with `FingerprintGenerator` that will populate several browser attributes to mimic real browser fingerprint. To read more about fingerprints please see: https://docs.apify.com/academy/anti-scraping/techniques/fingerprinting. - -You can implement your own fingerprint generator or use `DefaultFingerprintGenerator`. To use the generator initialize it with the desired fingerprint options. The generator will try to create fingerprint based on those options. Unspecified options will be automatically selected by the generator from the set of reasonable values. If some option is important for you, do not rely on the default and explicitly define it. - - - {PlaywrightCrawlerExample} - diff --git a/docs/examples/resuming_paused_crawl.mdx b/docs/examples/resuming_paused_crawl.mdx deleted file mode 100644 index 8d2213d11d..0000000000 --- a/docs/examples/resuming_paused_crawl.mdx +++ /dev/null @@ -1,35 +0,0 @@ ---- -id: resuming-paused-crawl -title: Resuming a paused crawl ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import ResumeCrawl from '!!raw-loader!roa-loader!./code_examples/resuming_paused_crawl.py'; - -This example demonstrates how to resume crawling from its last state when running locally, if for some reason it was unexpectedly terminated. - -If each run should continue crawling from the previous state, you can configure this using `purge_on_start` in `Configuration`. - -Use the code below and perform 2 sequential runs. During the 1st run, stop the crawler by pressing `CTRL+C`, and the 2nd run will resume crawling from where it stopped. - - - {ResumeCrawl} - - -Perform the 1st run, interrupting the crawler with `CTRL+C` after 2 links have been processed. - -![Run with interruption](/img/resuming-paused-crawl/00.webp 'Run with interruption.') - -Now resume crawling after the pause to process the remaining 3 links. - -![Resuming crawling](/img/resuming-paused-crawl/01.webp 'Resuming crawling.') - -Alternatively, use the environment variable `CRAWLEE_PURGE_ON_START=0` instead of using `configuration.purge_on_start = False`. - -For example, when running code: - -```bash -CRAWLEE_PURGE_ON_START=0 python -m best_crawler -``` diff --git a/docs/examples/using_browser_profile.mdx b/docs/examples/using_browser_profile.mdx deleted file mode 100644 index 8eda2554a4..0000000000 --- a/docs/examples/using_browser_profile.mdx +++ /dev/null @@ -1,39 +0,0 @@ ---- -id: using_browser_profile -title: Using browser profile ---- - -import ApiLink from '@site/src/components/ApiLink'; - -import CodeBlock from '@theme/CodeBlock'; - -import ChromeProfileExample from '!!raw-loader!./code_examples/using_browser_profiles_chrome.py'; -import FirefoxProfileExample from '!!raw-loader!./code_examples/using_browser_profiles_firefox.py'; - -This example demonstrates how to run `PlaywrightCrawler` using your local browser profile from [Chrome](https://www.google.com/intl/us/chrome/) or [Firefox](https://www.firefox.com/). - -Using browser profiles allows you to leverage existing login sessions, saved passwords, bookmarks, and other personalized browser data during crawling. This can be particularly useful for testing scenarios or when you need to access content that requires authentication. - -## Chrome browser - -To run `PlaywrightCrawler` with your Chrome profile, you need to know the path to your profile files. You can find this information by entering `chrome://version/` as a URL in your Chrome browser. If you have multiple profiles, pay attention to the profile name - if you only have one profile, it's always `Default`. - -:::warning Profile access limitation -Due to [Chrome's security policies](https://developer.chrome.com/blog/remote-debugging-port), automation cannot use your main browsing profile directly. The example copies your profile to a temporary location as a workaround. -::: - -Make sure you don't have any running Chrome browser processes before running this code: - - - {ChromeProfileExample} - - -## Firefox browser - -To find the path to your Firefox profile, enter `about:profiles` as a URL in your Firefox browser. Unlike Chrome, you can use your standard profile path directly without copying it first. - -Make sure you don't have any running Firefox browser processes before running this code: - - - {FirefoxProfileExample} - diff --git a/docs/examples/using_sitemap_request_loader.mdx b/docs/examples/using_sitemap_request_loader.mdx deleted file mode 100644 index 3ed528e94e..0000000000 --- a/docs/examples/using_sitemap_request_loader.mdx +++ /dev/null @@ -1,22 +0,0 @@ ---- -id: using-sitemap-request-loader -title: Using sitemap request loader ---- - -import ApiLink from '@site/src/components/ApiLink'; - -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import SitemapRequestLoaderExample from '!!raw-loader!roa-loader!./code_examples/using_sitemap_request_loader.py'; - -This example demonstrates how to use `SitemapRequestLoader` to crawl websites that provide `sitemap.xml` files following the [Sitemaps protocol](https://www.sitemaps.org/protocol.html). The `SitemapRequestLoader` processes sitemaps in a streaming fashion without loading them entirely into memory, making it suitable for large sitemaps. - -The example shows how to use the `transform_request_function` parameter to configure request options based on URL patterns. This allows you to modify request properties such as labels and user data based on the source URL, enabling different handling logic for different websites or sections. - -The following code example implements processing of sitemaps from two different domains (Apify and Crawlee), with different labels assigned to requests based on their host. The `create_transform_request` function maps each host to the corresponding request configuration, while the crawler uses different handlers based on the assigned labels. - - - {SitemapRequestLoaderExample} - - -For more information about request loaders, see the [Request loaders guide](../guides/request-loaders). diff --git a/pyproject.toml b/pyproject.toml index 5423ae359b..892ce7ea67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,16 +237,16 @@ indent-style = "space" "N999", # Invalid module name "T201", # `print` found ] -"**/docs/examples/code_examples/*crawler_with_error_snapshotter.py" = [ +"**/docs/03_concepts/code_examples/error_handling/*crawler_with_error_snapshotter.py" = [ "PLR2004", # Magic value used in comparison. Ignored for simplicity and readability of example code. ] -"**/docs/guides/code_examples/running_in_web_server/server.py" = [ +"**/docs/04_guides/code_examples/running_in_web_server/server.py" = [ "TC002", # ruff false positive. Import actually needed during runtime. ] -"**/docs/guides/code_examples/creating_web_archive/*.*" = [ +"**/docs/04_guides/code_examples/creating_web_archive/*.*" = [ "ASYNC230", # Ignore for simplicity of the example. ] -"**/docs/guides/code_examples/scrapy_migration/scrapy_*.py" = [ +"**/docs/04_guides/code_examples/scrapy_migration/scrapy_*.py" = [ "ANN", # Idiomatic Scrapy tutorial snippets are untyped; shown for comparison, not executed. "RUF012", # Scrapy declares `start_urls` as a plain class-level list. ] @@ -295,7 +295,7 @@ python-version = "3.10" include = ["src", "tests", "scripts", "docs", "website"] exclude = [ "src/crawlee/project_template", - "docs/guides/code_examples/storage_clients/custom_storage_client_example.py", + "docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py", "website/versioned_docs", ] diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 34d243e8a9..9fff82d8df 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -120,40 +120,112 @@ module.exports = { }, }, ], - // [ - // '@docusaurus/plugin-client-redirects', - // { - // redirects: [ - // { - // from: '/docs', - // to: '/docs/quick-start', - // }, - // { - // from: '/docs/next', - // to: '/docs/next/quick-start', - // }, - // { - // from: '/docs/guides/environment-variables', - // to: '/docs/guides/configuration', - // }, - // { - // from: '/docs/guides/getting-started', - // to: '/docs/introduction', - // }, - // { - // from: '/docs/guides/apify-platform', - // to: '/docs/deployment/apify-platform', - // }, - // ], - // createRedirects(existingPath) { - // if (!existingPath.endsWith('/')) { - // return `${existingPath}/`; - // } - // - // return undefined; // Return a falsy value: no redirect created - // }, - // }, - // ], + [ + '@docusaurus/plugin-client-redirects', + { + createRedirects(existingPath) { + // Maps a current docs path suffix to the old suffixes that should redirect + // to it, covering the restructuring that introduced the Concepts section + // and merged the Examples section into Concepts and Guides. Redirects are + // derived from existing routes, so they apply to every docs version that + // contains the new page and never shadow versions that still have the old + // layout (see OLD_STRUCTURE_VERSIONS below). + const MOVED_DOCS = { + 'concepts/architecture-overview': ['guides/architecture-overview'], + 'concepts/http-crawlers': [ + 'guides/http-crawlers', + 'examples/beautifulsoup-crawler', + 'examples/parsel-crawler', + 'examples/file-download', + ], + 'concepts/playwright-crawler': [ + 'guides/playwright-crawler', + 'examples/playwright-crawler', + 'examples/playwright-crawler-with-block-requests', + 'examples/capture-screenshots-using-playwright', + ], + 'concepts/adaptive-playwright-crawler': [ + 'guides/adaptive-playwright-crawler', + 'examples/adaptive-playwright-crawler', + ], + 'concepts/request-router': ['guides/request-router'], + 'concepts/request-loaders': ['guides/request-loaders', 'examples/using-sitemap-request-loader'], + 'concepts/storages': [ + 'guides/storages', + 'examples/add-data-to-dataset', + 'examples/export-entire-dataset-to-file', + ], + 'concepts/storage-clients': ['guides/storage-clients'], + 'concepts/http-clients': ['guides/http-clients'], + 'concepts/session-management': ['guides/session-management'], + 'concepts/cookie-management': ['guides/cookie-management'], + 'concepts/http-headers': ['guides/http-headers'], + 'concepts/proxy-management': ['guides/proxy-management'], + 'concepts/scaling-crawlers': ['guides/scaling-crawlers'], + 'concepts/request-throttling': ['guides/request-throttling'], + 'concepts/error-handling': [ + 'guides/error-handling', + 'examples/capturing-page-snapshots-with-error-snapshotter', + ], + 'concepts/logging': ['examples/configure-json-logging'], + 'concepts/service-locator': ['guides/service-locator'], + 'guides/crawling-links': [ + 'examples/crawl-all-links-on-website', + 'examples/crawl-multiple-urls', + 'examples/crawl-specific-links-on-website', + 'examples/crawl-website-with-relative-links', + ], + 'guides/fill-and-submit-web-form': ['examples/fill-and-submit-web-form'], + 'guides/respect-robots-txt-file': ['examples/respect-robots-txt-file'], + 'guides/stopping-and-resuming-crawlers': [ + 'examples/crawler-stop', + 'examples/crawler-keep-alive', + 'examples/resuming-paused-crawl', + ], + 'guides/run-parallel-crawlers': ['examples/run-parallel-crawlers'], + 'guides/avoid-blocking': [ + 'examples/playwright-crawler-with-camoufox', + 'examples/playwright-crawler-with-fingerprint-generator', + ], + 'guides/logging-in-with-a-crawler': ['examples/using_browser_profile'], + guides: ['examples'], + }; + + // Doc versions that still have the pre-restructure layout. Their routes + // (e.g. /docs/1.9/examples/...) must not get redirects generated over them. + const OLD_STRUCTURE_VERSIONS = ['0.6', '1.9']; + const versions = require('./versions.json'); + const latestHasOldStructure = OLD_STRUCTURE_VERSIONS.includes(versions[0]); + + const marker = '/docs/'; + const markerIndex = existingPath.indexOf(marker); + + if (markerIndex === -1) { + return undefined; + } + + const redirects = []; + + for (const [currentSuffix, oldSuffixes] of Object.entries(MOVED_DOCS)) { + if (!existingPath.endsWith(`/${currentSuffix}`)) { + continue; + } + + // E.g. '/python/docs/', '/python/docs/next/' or '/python/docs/1.9/'. + const docsRoot = existingPath.slice(0, existingPath.length - currentSuffix.length); + const version = docsRoot.slice(markerIndex + marker.length).replace(/\/$/, ''); + const hasOldStructure = + version === '' ? latestHasOldStructure : OLD_STRUCTURE_VERSIONS.includes(version); + + if (!hasOldStructure) { + redirects.push(...oldSuffixes.map((oldSuffix) => `${docsRoot}${oldSuffix}`)); + } + } + + return redirects.length > 0 ? redirects : undefined; + }, + }, + ], [ 'docusaurus-gtm-plugin', { @@ -288,12 +360,6 @@ module.exports = { label: 'Docs', position: 'left', }, - { - type: 'doc', - docId: '/examples', - label: 'Examples', - position: 'left', - }, { type: 'custom-api', label: 'API', @@ -351,12 +417,12 @@ module.exports = { title: 'Docs', items: [ { - label: 'Guides', - to: 'docs/guides', + label: 'Quick start', + to: 'docs/quick-start', }, { - label: 'Examples', - to: 'docs/examples', + label: 'Guides', + to: 'docs/guides', }, { label: 'API reference', diff --git a/website/sidebars.js b/website/sidebars.js index 8379abf15f..cab87b4881 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -21,6 +21,25 @@ module.exports = { 'introduction/deployment', ], }, + { + type: 'category', + label: 'Concepts', + collapsed: true, + link: { + type: 'generated-index', + title: 'Concepts', + description: + 'Learn how the core components of Crawlee work - crawlers, request routing, storages, sessions, and more.', + slug: '/concepts', + keywords: ['concepts'], + }, + items: [ + { + type: 'autogenerated', + dirName: '03_concepts', + }, + ], + }, { type: 'category', label: 'Guides', @@ -28,13 +47,15 @@ module.exports = { link: { type: 'generated-index', title: 'Guides', + description: + 'Task-oriented guides for common crawling and scraping scenarios, integrations, and migrations.', slug: '/guides', keywords: ['guides'], }, items: [ { type: 'autogenerated', - dirName: 'guides', + dirName: '04_guides', }, ], }, @@ -66,39 +87,6 @@ module.exports = { }, ], }, - { - type: 'category', - label: 'Examples', - collapsed: true, - link: { - type: 'generated-index', - title: 'Examples', - slug: '/examples', - keywords: ['examples'], - }, - items: [ - { - type: 'autogenerated', - dirName: 'examples', - }, - ], - }, - // { - // type: 'category', - // label: 'Experiments', - // link: { - // type: 'generated-index', - // title: 'Experiments', - // slug: '/experiments', - // keywords: ['experiments', 'experimental-features'], - // }, - // items: [ - // { - // type: 'autogenerated', - // dirName: 'experiments', - // }, - // ], - // }, { type: 'category', label: 'Upgrading', @@ -112,7 +100,7 @@ module.exports = { items: [ { type: 'autogenerated', - dirName: 'upgrading', + dirName: '06_upgrading', }, ], }, diff --git a/website/src/pages/index.js b/website/src/pages/index.js index b4c6d58ccd..1867602759 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -107,13 +107,15 @@ function BenefitsSection() { ); } +// The /docs/next prefix on the concepts links is temporary: the Concepts section exists only in the +// next (unreleased) docs until the first post-restructure version snapshot. Drop the prefix then. function OtherFeaturesSection() { return (

What else is in Crawlee?

- +
- + Date: Fri, 16 Oct 2026 12:28:58 +0200 Subject: [PATCH 02/11] docs: reorganize sections into introduction, concepts, guides, and upgrading --- README.md | 2 +- .../01_setting_up.mdx | 0 .../02_first_crawler.mdx | 0 .../03_adding_more_urls.mdx | 0 .../04_real_world_project.mdx | 0 .../05_crawling.mdx | 0 .../06_scraping.mdx | 0 .../07_saving_data.mdx | 0 .../08_refactoring.mdx | 2 +- .../09_running_in_cloud.mdx | 2 +- .../code_examples/02_bs.py | 0 .../code_examples/02_bs_better.py | 0 .../code_examples/02_request_queue.py | 0 .../code_examples/03_enqueue_strategy.py | 0 .../code_examples/03_finding_new_links.py | 0 .../code_examples/03_globs.py | 0 .../code_examples/03_original_code.py | 0 .../code_examples/03_transform_request.py | 0 .../code_examples/04_sanity_check.py | 0 .../code_examples/05_crawling_detail.py | 0 .../code_examples/05_crawling_listing.py | 0 .../code_examples/06_scraping.py | 0 .../code_examples/07_final_code.py | 0 .../code_examples/07_first_code.py | 0 .../code_examples/08_main.py | 0 .../code_examples/08_routes.py | 0 .../code_examples/09_apify_sdk.py | 0 .../code_examples/__init__.py | 0 .../beautifulsoup_crawler_example.py | 0 .../quick_start}/parsel_crawler_example.py | 0 .../playwright_crawler_example.py | 0 .../playwright_crawler_headful_example.py | 0 .../code_examples/routes.py | 0 .../index.mdx | 0 .../quick-start.mdx} | 15 ++--- .../01_architecture_overview.mdx | 2 +- .../04_request_router.mdx} | 0 .../05_http_crawlers.mdx} | 0 .../06_playwright_crawler.mdx} | 0 .../07_playwright_crawler_adaptive.mdx} | 0 .../08_pydantic_ai_crawler.mdx} | 0 .../09_stagehand_crawler.mdx} | 2 +- .../10_request_loaders.mdx} | 0 .../11_storages.mdx} | 0 .../12_storage_clients.mdx} | 0 .../13_http_clients.mdx} | 0 .../14_session_management.mdx} | 0 .../15_cookie_management.mdx} | 0 .../16_http_headers.mdx} | 0 .../17_proxy_management.mdx} | 0 .../18_scaling_crawlers.mdx} | 0 .../19_request_throttling.mdx} | 0 .../20_error_handling.mdx} | 0 .../23_logging.mdx} | 0 .../25_service_locator.mdx} | 0 .../26_extending_crawlee.mdx} | 8 +-- .../cookie_management/initial_cookies.py | 0 .../cookie_management/persist_cookies.py | 0 .../cookie_management/playwright_cookies.py | 0 .../cookie_management/read_write_cookies.py | 0 .../cookie_management/retry_pinned_session.py | 0 .../retry_restore_cookies.py | 0 .../cookie_management/retry_single_session.py | 0 .../change_handle_error_status.py | 0 .../error_handling/disable_retry.py | 0 .../error_handling/handle_proxy_error.py | 0 .../parsel_crawler_with_error_snapshotter.py | 0 ...aywright_crawler_with_error_snapshotter.py | 0 .../parsel_curl_impersonate_example.py | 0 .../http_clients/parsel_httpx_example.py | 0 .../http_clients/parsel_impit_example.py | 0 .../code_examples/http_crawlers/__init__.py | 0 .../http_crawlers/beautifulsoup_example.py | 0 .../http_crawlers/custom_crawler_example.py | 0 .../http_crawlers/file_download.py | 0 .../http_crawlers/file_download_stream.py | 0 .../http_crawlers/http_example.py | 0 .../http_crawlers/lexbor_parser.py | 0 .../http_crawlers/lxml_parser.py | 0 .../http_crawlers/lxml_saxonche_parser.py | 0 .../http_crawlers/parsel_example.py | 0 .../http_crawlers/pyquery_parser.py | 0 .../http_crawlers/scrapling_parser.py | 0 .../http_crawlers/selectolax_adaptive_run.py | 0 .../http_crawlers/selectolax_context.py | 0 .../http_crawlers/selectolax_crawler.py | 0 .../http_crawlers/selectolax_crawler_run.py | 0 .../http_crawlers/selectolax_parser.py | 0 .../http_headers/browser_page_headers.py | 0 .../code_examples/http_headers/set_headers.py | 0 .../logging/configure_json_logging.py | 0 .../browser_configuration_example.py | 0 .../browser_pool_launch_hooks_example.py | 0 .../browser_pool_page_hooks_example.py | 0 .../capture_screenshot_using_playwright.py | 0 .../multiple_launch_example.py | 0 .../navigation_hooks_example.py | 0 .../playwright_block_requests.py | 0 .../playwright_crawler/playwright_crawler.py | 0 .../plugin_browser_configuration_example.py | 0 .../adaptive_playwright_crawler.py | 0 .../playwright_crawler_adaptive/handler.py | 0 .../init_beautifulsoup.py | 0 .../init_parsel.py | 0 .../init_prediction.py | 0 .../pre_nav_hooks.py | 0 .../proxy_management/inspecting_bs_example.py | 0 .../proxy_management/inspecting_pw_example.py | 0 .../integration_bs_example.py | 0 .../integration_pw_example.py | 0 .../proxy_management/quick_start_example.py | 0 .../proxy_management/session_bs_example.py | 0 .../proxy_management/session_pw_example.py | 0 .../proxy_management/tiers_bs_example.py | 0 .../proxy_management/tiers_pw_example.py | 0 .../additional_instructions_example.py | 0 .../pydantic_ai_crawler/basic_example.py | 0 .../custom_distiller_example.py | 0 .../pydantic_ai_crawler/debugging_example.py | 0 .../selector_extractor_example.py | 0 .../usage_limit_example.py | 0 .../request_loaders/rl_basic_example.py | 0 .../rl_basic_example_with_persist.py | 0 .../request_loaders/rl_tandem_example.py | 0 .../rl_tandem_example_explicit.py | 0 .../request_loaders/sitemap_basic_example.py | 0 .../sitemap_example_with_persist.py | 0 .../request_loaders/sitemap_tandem_example.py | 0 .../sitemap_tandem_example_explicit.py | 0 .../using_sitemap_request_loader.py | 0 .../adaptive_crawler_handlers.py | 0 .../request_router/basic_request_handlers.py | 0 .../custom_router_default_only.py | 0 .../request_router/error_handler.py | 0 .../request_router/failed_request_handler.py | 0 .../request_router/http_pre_navigation.py | 0 .../playwright_pre_navigation.py | 0 .../request_router/router_middleware.py | 0 .../request_router/simple_default_handler.py | 0 .../request_throttling/throttling_example.py | 0 .../max_tasks_per_minute_example.py | 0 .../min_and_max_concurrency_example.py | 0 .../service_locator/service_conflicts.py | 0 .../service_crawler_configuration.py | 0 .../service_crawler_event_manager.py | 0 .../service_crawler_storage_client.py | 0 .../service_locator_configuration.py | 0 .../service_locator_event_manager.py | 0 .../service_locator_storage_client.py | 0 .../service_storage_configuration.py | 0 .../service_storage_storage_client.py | 0 .../session_management/multi_sessions_http.py | 0 .../session_management/one_session_http.py | 0 .../session_management/sm_basic.py | 0 .../session_management/sm_beautifulsoup.py | 0 .../session_management/sm_http.py | 0 .../session_management/sm_parsel.py | 0 .../session_management/sm_playwright.py | 0 .../session_management/sm_standalone.py | 0 .../stagehand_crawler/basic_example.py | 0 .../stagehand_crawler/browserbase_example.py | 0 .../custom_storage_client_example.py | 0 ...ile_system_storage_client_basic_example.py | 0 ...em_storage_client_configuration_example.py | 0 .../memory_storage_client_basic_example.py | 0 .../redis_storage_client_basic_example.py | 0 ...is_storage_client_configuration_example.py | 0 .../registering_storage_clients_example.py | 0 .../sql_storage_client_basic_example.py | 0 ...ql_storage_client_configuration_example.py | 0 .../storages/cleaning_do_not_purge_example.py | 0 .../cleaning_purge_explicitly_example.py | 0 .../storages/dataset_basic_example.py | 0 .../storages/dataset_with_crawler_example.py | 0 .../dataset_with_crawler_explicit_example.py | 0 .../export_entire_dataset_to_file_csv.py | 0 .../export_entire_dataset_to_file_json.py | 0 .../storages/helper_add_requests_example.py | 0 .../storages/helper_enqueue_links_example.py | 0 .../storages/kvs_basic_example.py | 0 .../storages/kvs_with_crawler_example.py | 0 .../kvs_with_crawler_explicit_example.py | 0 .../code_examples/storages/opening.py | 0 .../storages/rq_basic_example.py | 0 .../storages/rq_with_crawler_example.py | 0 .../rq_with_crawler_explicit_example.py | 0 .../01_crawling_links.mdx | 0 .../02_fill_and_submit_web_form.mdx | 0 .../03_crawler_login.mdx | 0 .../04_avoid_blocking.mdx | 0 .../05_respect_robots_txt_file.mdx | 0 .../06_stopping_and_resuming.mdx | 0 .../07_run_parallel_crawlers.mdx | 0 .../08_running_in_web_server.mdx | 0 .../09_trace_and_monitor_crawlers.mdx | 0 .../10_creating_web_archive.mdx | 0 .../11_secure_scraping.mdx | 2 +- .../12_scrapy_migration.mdx} | 0 .../13_deploy_on_apify.mdx} | 16 +++--- .../14_deploy_on_aws_lambda.mdx} | 10 ++-- .../15_deploy_on_gcp_cloud_run_functions.mdx} | 6 +- .../16_deploy_on_gcp_cloud_run.mdx} | 8 +-- ...default_fingerprint_generator_with_args.py | 0 .../playwright_crawler_with_camoufox.py | 0 .../playwright_with_cloakbrowser.py | 0 .../playwright_with_fingerprint_generator.py | 0 .../crawl_all_links_on_website_bs.py | 0 .../crawl_all_links_on_website_pw.py | 0 .../crawling_links/crawl_multiple_urls_bs.py | 0 .../crawling_links/crawl_multiple_urls_pw.py | 0 .../crawl_specific_links_on_website_bs.py | 0 .../crawl_specific_links_on_website_pw.py | 0 ...l_website_with_relative_links_all_links.py | 0 ...website_with_relative_links_same_domain.py | 0 ...bsite_with_relative_links_same_hostname.py | 0 ...website_with_relative_links_same_origin.py | 0 ...ct_and_add_specific_links_on_website_bs.py | 0 ...ct_and_add_specific_links_on_website_pw.py | 0 .../manual_archiving_parsel_crawler.py | 0 .../manual_archiving_playwright_crawler.py | 0 .../simple_pw_through_proxy_pywb_server.py | 0 .../apify/crawler_as_actor_example.py | 0 .../deployment}/apify/get_public_url.py | 0 .../apify/log_with_config_example.py | 0 .../apify/proxy_advanced_example.py | 0 .../deployment}/apify/proxy_example.py | 0 .../aws/beautifulsoup_crawler_lambda.py | 0 .../aws/playwright_crawler_lambda.py | 0 .../deployment}/aws/playwright_dockerfile | 0 .../deployment}/google/cloud_run_example.py | 0 .../deployment}/google/google_example.py | 0 .../fill_and_submit_web_form_automated.py | 0 .../fill_and_submit_web_form_crawler.py | 0 .../fill_and_submit_web_form_request.py | 0 .../code_examples/login_crawler/http_login.py | 0 .../login_crawler/playwright_login.py | 0 .../using_browser_profiles_chrome.py | 0 .../using_browser_profiles_firefox.py | 0 .../respect_robots_on_skipped_request.py | 0 .../respect_robots_txt_file.py | 0 .../run_parallel_crawlers.py | 0 .../running_in_web_server/__init__.py | 0 .../running_in_web_server/crawler.py | 0 .../running_in_web_server/server.py | 0 .../scrapy_migration/crawlee_concurrency.py | 0 .../scrapy_migration/crawlee_crawlspider.py | 0 .../crawlee_error_handling.py | 0 .../scrapy_migration/crawlee_export.py | 0 .../scrapy_migration/crawlee_labels.py | 0 .../scrapy_migration/crawlee_playwright.py | 0 .../scrapy_migration/crawlee_post.py | 0 .../scrapy_migration/crawlee_proxy.py | 0 .../scrapy_migration/crawlee_quotes.py | 0 .../scrapy_migration/crawlee_throttling.py | 0 .../scrapy_migration/crawlee_user_data.py | 0 .../scrapy_migration/scrapy_authors.py | 0 .../scrapy_migration/scrapy_cb_kwargs.py | 0 .../scrapy_migration/scrapy_concurrency.py | 0 .../scrapy_migration/scrapy_crawlspider.py | 0 .../scrapy_migration/scrapy_errback.py | 0 .../scrapy_migration/scrapy_export.py | 0 .../scrapy_migration/scrapy_formrequest.py | 0 .../scrapy_migration/scrapy_playwright.py | 0 .../scrapy_playwright_settings.py | 0 .../scrapy_migration/scrapy_proxy.py | 0 .../scrapy_migration/scrapy_quotes.py | 0 .../scrapy_migration/scrapy_throttling.py | 0 .../beautifulsoup_crawler_keep_alive.py | 0 .../beautifulsoup_crawler_stop.py | 0 .../resuming_paused_crawl.py | 0 .../instrument_crawler.py | 0 .../upgrading_to_v0x.md | 0 .../upgrading_to_v1.md | 0 pyproject.toml | 10 ++-- website/docusaurus.config.js | 20 +++++-- website/sidebars.js | 57 +++++++------------ 276 files changed, 78 insertions(+), 84 deletions(-) rename docs/{02_introduction => 01_introduction}/01_setting_up.mdx (100%) rename docs/{02_introduction => 01_introduction}/02_first_crawler.mdx (100%) rename docs/{02_introduction => 01_introduction}/03_adding_more_urls.mdx (100%) rename docs/{02_introduction => 01_introduction}/04_real_world_project.mdx (100%) rename docs/{02_introduction => 01_introduction}/05_crawling.mdx (100%) rename docs/{02_introduction => 01_introduction}/06_scraping.mdx (100%) rename docs/{02_introduction => 01_introduction}/07_saving_data.mdx (100%) rename docs/{02_introduction => 01_introduction}/08_refactoring.mdx (97%) rename docs/{02_introduction => 01_introduction}/09_running_in_cloud.mdx (99%) rename docs/{02_introduction => 01_introduction}/code_examples/02_bs.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/02_bs_better.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/02_request_queue.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/03_enqueue_strategy.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/03_finding_new_links.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/03_globs.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/03_original_code.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/03_transform_request.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/04_sanity_check.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/05_crawling_detail.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/05_crawling_listing.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/06_scraping.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/07_final_code.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/07_first_code.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/08_main.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/08_routes.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/09_apify_sdk.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/__init__.py (100%) rename docs/{01_quick-start/code_examples => 01_introduction/code_examples/quick_start}/beautifulsoup_crawler_example.py (100%) rename docs/{01_quick-start/code_examples => 01_introduction/code_examples/quick_start}/parsel_crawler_example.py (100%) rename docs/{01_quick-start/code_examples => 01_introduction/code_examples/quick_start}/playwright_crawler_example.py (100%) rename docs/{01_quick-start/code_examples => 01_introduction/code_examples/quick_start}/playwright_crawler_headful_example.py (100%) rename docs/{02_introduction => 01_introduction}/code_examples/routes.py (100%) rename docs/{02_introduction => 01_introduction}/index.mdx (100%) rename docs/{01_quick-start/index.mdx => 01_introduction/quick-start.mdx} (91%) rename docs/{03_concepts => 02_concepts}/01_architecture_overview.mdx (99%) rename docs/{03_concepts/05_request_router.mdx => 02_concepts/04_request_router.mdx} (100%) rename docs/{03_concepts/02_http_crawlers.mdx => 02_concepts/05_http_crawlers.mdx} (100%) rename docs/{03_concepts/03_playwright_crawler.mdx => 02_concepts/06_playwright_crawler.mdx} (100%) rename docs/{03_concepts/04_playwright_crawler_adaptive.mdx => 02_concepts/07_playwright_crawler_adaptive.mdx} (100%) rename docs/{04_guides/13_pydantic_ai_crawler.mdx => 02_concepts/08_pydantic_ai_crawler.mdx} (100%) rename docs/{04_guides/14_stagehand_crawler.mdx => 02_concepts/09_stagehand_crawler.mdx} (99%) rename docs/{03_concepts/06_request_loaders.mdx => 02_concepts/10_request_loaders.mdx} (100%) rename docs/{03_concepts/07_storages.mdx => 02_concepts/11_storages.mdx} (100%) rename docs/{03_concepts/08_storage_clients.mdx => 02_concepts/12_storage_clients.mdx} (100%) rename docs/{03_concepts/09_http_clients.mdx => 02_concepts/13_http_clients.mdx} (100%) rename docs/{03_concepts/10_session_management.mdx => 02_concepts/14_session_management.mdx} (100%) rename docs/{03_concepts/11_cookie_management.mdx => 02_concepts/15_cookie_management.mdx} (100%) rename docs/{03_concepts/12_http_headers.mdx => 02_concepts/16_http_headers.mdx} (100%) rename docs/{03_concepts/13_proxy_management.mdx => 02_concepts/17_proxy_management.mdx} (100%) rename docs/{03_concepts/14_scaling_crawlers.mdx => 02_concepts/18_scaling_crawlers.mdx} (100%) rename docs/{03_concepts/15_request_throttling.mdx => 02_concepts/19_request_throttling.mdx} (100%) rename docs/{03_concepts/16_error_handling.mdx => 02_concepts/20_error_handling.mdx} (100%) rename docs/{03_concepts/17_logging.mdx => 02_concepts/23_logging.mdx} (100%) rename docs/{03_concepts/18_service_locator.mdx => 02_concepts/25_service_locator.mdx} (100%) rename docs/{04_guides/12_extending_crawlee.mdx => 02_concepts/26_extending_crawlee.mdx} (93%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/initial_cookies.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/persist_cookies.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/playwright_cookies.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/read_write_cookies.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/retry_pinned_session.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/retry_restore_cookies.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/cookie_management/retry_single_session.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/error_handling/change_handle_error_status.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/error_handling/disable_retry.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/error_handling/handle_proxy_error.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_clients/parsel_curl_impersonate_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_clients/parsel_httpx_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_clients/parsel_impit_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/__init__.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/beautifulsoup_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/custom_crawler_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/file_download.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/file_download_stream.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/http_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/lexbor_parser.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/lxml_parser.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/lxml_saxonche_parser.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/parsel_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/pyquery_parser.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/scrapling_parser.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/selectolax_adaptive_run.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/selectolax_context.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/selectolax_crawler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/selectolax_crawler_run.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_crawlers/selectolax_parser.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_headers/browser_page_headers.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/http_headers/set_headers.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/logging/configure_json_logging.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/browser_configuration_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/browser_pool_page_hooks_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/capture_screenshot_using_playwright.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/multiple_launch_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/navigation_hooks_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/playwright_block_requests.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/playwright_crawler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler/plugin_browser_configuration_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler_adaptive/handler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler_adaptive/init_parsel.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler_adaptive/init_prediction.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/inspecting_bs_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/inspecting_pw_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/integration_bs_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/integration_pw_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/quick_start_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/session_bs_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/session_pw_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/tiers_bs_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/proxy_management/tiers_pw_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/pydantic_ai_crawler/additional_instructions_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/pydantic_ai_crawler/basic_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/pydantic_ai_crawler/custom_distiller_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/pydantic_ai_crawler/debugging_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/pydantic_ai_crawler/selector_extractor_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/pydantic_ai_crawler/usage_limit_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/rl_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/rl_basic_example_with_persist.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/rl_tandem_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/rl_tandem_example_explicit.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/sitemap_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/sitemap_example_with_persist.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/sitemap_tandem_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/sitemap_tandem_example_explicit.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_loaders/using_sitemap_request_loader.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/adaptive_crawler_handlers.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/basic_request_handlers.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/custom_router_default_only.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/error_handler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/failed_request_handler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/http_pre_navigation.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/playwright_pre_navigation.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/router_middleware.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_router/simple_default_handler.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/request_throttling/throttling_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/scaling_crawlers/max_tasks_per_minute_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/scaling_crawlers/min_and_max_concurrency_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_conflicts.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_crawler_configuration.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_crawler_event_manager.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_crawler_storage_client.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_locator_configuration.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_locator_event_manager.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_locator_storage_client.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_storage_configuration.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/service_locator/service_storage_storage_client.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/multi_sessions_http.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/one_session_http.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/sm_basic.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/sm_beautifulsoup.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/sm_http.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/sm_parsel.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/sm_playwright.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/session_management/sm_standalone.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/stagehand_crawler/basic_example.py (100%) rename docs/{04_guides => 02_concepts}/code_examples/stagehand_crawler/browserbase_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/custom_storage_client_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/file_system_storage_client_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/file_system_storage_client_configuration_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/memory_storage_client_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/redis_storage_client_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/redis_storage_client_configuration_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/registering_storage_clients_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/sql_storage_client_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storage_clients/sql_storage_client_configuration_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/cleaning_do_not_purge_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/cleaning_purge_explicitly_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/dataset_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/dataset_with_crawler_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/dataset_with_crawler_explicit_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/export_entire_dataset_to_file_csv.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/export_entire_dataset_to_file_json.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/helper_add_requests_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/helper_enqueue_links_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/kvs_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/kvs_with_crawler_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/kvs_with_crawler_explicit_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/opening.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/rq_basic_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/rq_with_crawler_example.py (100%) rename docs/{03_concepts => 02_concepts}/code_examples/storages/rq_with_crawler_explicit_example.py (100%) rename docs/{04_guides => 03_guides}/01_crawling_links.mdx (100%) rename docs/{04_guides => 03_guides}/02_fill_and_submit_web_form.mdx (100%) rename docs/{04_guides => 03_guides}/03_crawler_login.mdx (100%) rename docs/{04_guides => 03_guides}/04_avoid_blocking.mdx (100%) rename docs/{04_guides => 03_guides}/05_respect_robots_txt_file.mdx (100%) rename docs/{04_guides => 03_guides}/06_stopping_and_resuming.mdx (100%) rename docs/{04_guides => 03_guides}/07_run_parallel_crawlers.mdx (100%) rename docs/{04_guides => 03_guides}/08_running_in_web_server.mdx (100%) rename docs/{04_guides => 03_guides}/09_trace_and_monitor_crawlers.mdx (100%) rename docs/{04_guides => 03_guides}/10_creating_web_archive.mdx (100%) rename docs/{04_guides => 03_guides}/11_secure_scraping.mdx (99%) rename docs/{04_guides/15_scrapy_migration.mdx => 03_guides/12_scrapy_migration.mdx} (100%) rename docs/{05_deployment/apify_platform.mdx => 03_guides/13_deploy_on_apify.mdx} (95%) rename docs/{05_deployment/aws_lambda.mdx => 03_guides/14_deploy_on_aws_lambda.mdx} (97%) rename docs/{05_deployment/google_cloud.mdx => 03_guides/15_deploy_on_gcp_cloud_run_functions.mdx} (92%) rename docs/{05_deployment/google_cloud_run.mdx => 03_guides/16_deploy_on_gcp_cloud_run.mdx} (91%) rename docs/{04_guides => 03_guides}/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py (100%) rename docs/{04_guides => 03_guides}/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py (100%) rename docs/{04_guides => 03_guides}/code_examples/avoid_blocking/playwright_with_cloakbrowser.py (100%) rename docs/{04_guides => 03_guides}/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_all_links_on_website_bs.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_all_links_on_website_pw.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_multiple_urls_bs.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_multiple_urls_pw.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_specific_links_on_website_bs.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_specific_links_on_website_pw.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py (100%) rename docs/{04_guides => 03_guides}/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py (100%) rename docs/{04_guides => 03_guides}/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py (100%) rename docs/{04_guides => 03_guides}/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py (100%) rename docs/{04_guides => 03_guides}/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/apify/crawler_as_actor_example.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/apify/get_public_url.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/apify/log_with_config_example.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/apify/proxy_advanced_example.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/apify/proxy_example.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/aws/beautifulsoup_crawler_lambda.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/aws/playwright_crawler_lambda.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/aws/playwright_dockerfile (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/google/cloud_run_example.py (100%) rename docs/{05_deployment/code_examples => 03_guides/code_examples/deployment}/google/google_example.py (100%) rename docs/{04_guides => 03_guides}/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py (100%) rename docs/{04_guides => 03_guides}/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py (100%) rename docs/{04_guides => 03_guides}/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py (100%) rename docs/{04_guides => 03_guides}/code_examples/login_crawler/http_login.py (100%) rename docs/{04_guides => 03_guides}/code_examples/login_crawler/playwright_login.py (100%) rename docs/{04_guides => 03_guides}/code_examples/login_crawler/using_browser_profiles_chrome.py (100%) rename docs/{04_guides => 03_guides}/code_examples/login_crawler/using_browser_profiles_firefox.py (100%) rename docs/{04_guides => 03_guides}/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py (100%) rename docs/{04_guides => 03_guides}/code_examples/respect_robots_txt_file/respect_robots_txt_file.py (100%) rename docs/{04_guides => 03_guides}/code_examples/run_parallel_crawlers/run_parallel_crawlers.py (100%) rename docs/{04_guides => 03_guides}/code_examples/running_in_web_server/__init__.py (100%) rename docs/{04_guides => 03_guides}/code_examples/running_in_web_server/crawler.py (100%) rename docs/{04_guides => 03_guides}/code_examples/running_in_web_server/server.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_concurrency.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_crawlspider.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_error_handling.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_export.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_labels.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_playwright.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_post.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_proxy.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_quotes.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_throttling.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/crawlee_user_data.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_authors.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_cb_kwargs.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_concurrency.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_crawlspider.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_errback.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_export.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_formrequest.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_playwright.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_playwright_settings.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_proxy.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_quotes.py (100%) rename docs/{04_guides => 03_guides}/code_examples/scrapy_migration/scrapy_throttling.py (100%) rename docs/{04_guides => 03_guides}/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py (100%) rename docs/{04_guides => 03_guides}/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py (100%) rename docs/{04_guides => 03_guides}/code_examples/stopping_and_resuming/resuming_paused_crawl.py (100%) rename docs/{04_guides => 03_guides}/code_examples/trace_and_monitor_crawlers/instrument_crawler.py (100%) rename docs/{06_upgrading => 04_upgrading}/upgrading_to_v0x.md (100%) rename docs/{06_upgrading => 04_upgrading}/upgrading_to_v1.md (100%) diff --git a/README.md b/README.md index 6e33b33fab..e9d4a43b64 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ We also have a TypeScript implementation of the Crawlee, which you can explore a ## Installation -We recommend visiting the [Introduction tutorial](https://crawlee.dev/python/docs/introduction) in Crawlee documentation for more information. +For a step-by-step walkthrough, see the [tutorial](https://crawlee.dev/python/docs/introduction/tutorial) in the Crawlee documentation. Crawlee is available as [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. This package includes the core functionality, while additional features are available as optional extras to keep dependencies and package size minimal. diff --git a/docs/02_introduction/01_setting_up.mdx b/docs/01_introduction/01_setting_up.mdx similarity index 100% rename from docs/02_introduction/01_setting_up.mdx rename to docs/01_introduction/01_setting_up.mdx diff --git a/docs/02_introduction/02_first_crawler.mdx b/docs/01_introduction/02_first_crawler.mdx similarity index 100% rename from docs/02_introduction/02_first_crawler.mdx rename to docs/01_introduction/02_first_crawler.mdx diff --git a/docs/02_introduction/03_adding_more_urls.mdx b/docs/01_introduction/03_adding_more_urls.mdx similarity index 100% rename from docs/02_introduction/03_adding_more_urls.mdx rename to docs/01_introduction/03_adding_more_urls.mdx diff --git a/docs/02_introduction/04_real_world_project.mdx b/docs/01_introduction/04_real_world_project.mdx similarity index 100% rename from docs/02_introduction/04_real_world_project.mdx rename to docs/01_introduction/04_real_world_project.mdx diff --git a/docs/02_introduction/05_crawling.mdx b/docs/01_introduction/05_crawling.mdx similarity index 100% rename from docs/02_introduction/05_crawling.mdx rename to docs/01_introduction/05_crawling.mdx diff --git a/docs/02_introduction/06_scraping.mdx b/docs/01_introduction/06_scraping.mdx similarity index 100% rename from docs/02_introduction/06_scraping.mdx rename to docs/01_introduction/06_scraping.mdx diff --git a/docs/02_introduction/07_saving_data.mdx b/docs/01_introduction/07_saving_data.mdx similarity index 100% rename from docs/02_introduction/07_saving_data.mdx rename to docs/01_introduction/07_saving_data.mdx diff --git a/docs/02_introduction/08_refactoring.mdx b/docs/01_introduction/08_refactoring.mdx similarity index 97% rename from docs/02_introduction/08_refactoring.mdx rename to docs/01_introduction/08_refactoring.mdx index 7245e576f3..8118ae9e06 100644 --- a/docs/02_introduction/08_refactoring.mdx +++ b/docs/01_introduction/08_refactoring.mdx @@ -70,4 +70,4 @@ It's good practice in any programming language to split your logic into bite-siz ## Next steps -In the next and final step, you'll see how to deploy your Crawlee project to the cloud. If you used the CLI to bootstrap your project, you already have a `Dockerfile` ready, and the next section will show you how to deploy it to the [Apify platform](../deployment/apify-platform) with ease. +In the next and final step, you'll see how to deploy your Crawlee project to the cloud. If you used the CLI to bootstrap your project, you already have a `Dockerfile` ready, and the next section will show you how to deploy it to the [Apify platform](../guides/deploy-on-apify) with ease. diff --git a/docs/02_introduction/09_running_in_cloud.mdx b/docs/01_introduction/09_running_in_cloud.mdx similarity index 99% rename from docs/02_introduction/09_running_in_cloud.mdx rename to docs/01_introduction/09_running_in_cloud.mdx index c6fb8a5859..e3523a4758 100644 --- a/docs/02_introduction/09_running_in_cloud.mdx +++ b/docs/01_introduction/09_running_in_cloud.mdx @@ -12,7 +12,7 @@ import MainExample from '!!raw-loader!./code_examples/09_apify_sdk.py'; Crawlee is developed by [**Apify**](https://apify.com), the web scraping and automation platform. You could say it is the **home of Crawlee projects**. In this section you'll see how to deploy the crawler there with just a few simple steps. You can deploy a **Crawlee** project wherever you want, but using the [**Apify platform**](https://console.apify.com) will give you the best experience. -{/*In case you want to deploy your Crawlee project to other platforms, check out the [**Deployment**](../deployment) section.*/} +{/*In case you want to deploy your Crawlee project to other platforms, check out the [**Deployment**](../guides) section.*/} With a few simple steps, you can convert your Crawlee project into a so-called **Actor**. Actors are serverless micro-apps that are easy to develop, run, share, and integrate. The infra, proxies, and storages are ready to go. [Learn more about Actors](https://apify.com/actors). diff --git a/docs/02_introduction/code_examples/02_bs.py b/docs/01_introduction/code_examples/02_bs.py similarity index 100% rename from docs/02_introduction/code_examples/02_bs.py rename to docs/01_introduction/code_examples/02_bs.py diff --git a/docs/02_introduction/code_examples/02_bs_better.py b/docs/01_introduction/code_examples/02_bs_better.py similarity index 100% rename from docs/02_introduction/code_examples/02_bs_better.py rename to docs/01_introduction/code_examples/02_bs_better.py diff --git a/docs/02_introduction/code_examples/02_request_queue.py b/docs/01_introduction/code_examples/02_request_queue.py similarity index 100% rename from docs/02_introduction/code_examples/02_request_queue.py rename to docs/01_introduction/code_examples/02_request_queue.py diff --git a/docs/02_introduction/code_examples/03_enqueue_strategy.py b/docs/01_introduction/code_examples/03_enqueue_strategy.py similarity index 100% rename from docs/02_introduction/code_examples/03_enqueue_strategy.py rename to docs/01_introduction/code_examples/03_enqueue_strategy.py diff --git a/docs/02_introduction/code_examples/03_finding_new_links.py b/docs/01_introduction/code_examples/03_finding_new_links.py similarity index 100% rename from docs/02_introduction/code_examples/03_finding_new_links.py rename to docs/01_introduction/code_examples/03_finding_new_links.py diff --git a/docs/02_introduction/code_examples/03_globs.py b/docs/01_introduction/code_examples/03_globs.py similarity index 100% rename from docs/02_introduction/code_examples/03_globs.py rename to docs/01_introduction/code_examples/03_globs.py diff --git a/docs/02_introduction/code_examples/03_original_code.py b/docs/01_introduction/code_examples/03_original_code.py similarity index 100% rename from docs/02_introduction/code_examples/03_original_code.py rename to docs/01_introduction/code_examples/03_original_code.py diff --git a/docs/02_introduction/code_examples/03_transform_request.py b/docs/01_introduction/code_examples/03_transform_request.py similarity index 100% rename from docs/02_introduction/code_examples/03_transform_request.py rename to docs/01_introduction/code_examples/03_transform_request.py diff --git a/docs/02_introduction/code_examples/04_sanity_check.py b/docs/01_introduction/code_examples/04_sanity_check.py similarity index 100% rename from docs/02_introduction/code_examples/04_sanity_check.py rename to docs/01_introduction/code_examples/04_sanity_check.py diff --git a/docs/02_introduction/code_examples/05_crawling_detail.py b/docs/01_introduction/code_examples/05_crawling_detail.py similarity index 100% rename from docs/02_introduction/code_examples/05_crawling_detail.py rename to docs/01_introduction/code_examples/05_crawling_detail.py diff --git a/docs/02_introduction/code_examples/05_crawling_listing.py b/docs/01_introduction/code_examples/05_crawling_listing.py similarity index 100% rename from docs/02_introduction/code_examples/05_crawling_listing.py rename to docs/01_introduction/code_examples/05_crawling_listing.py diff --git a/docs/02_introduction/code_examples/06_scraping.py b/docs/01_introduction/code_examples/06_scraping.py similarity index 100% rename from docs/02_introduction/code_examples/06_scraping.py rename to docs/01_introduction/code_examples/06_scraping.py diff --git a/docs/02_introduction/code_examples/07_final_code.py b/docs/01_introduction/code_examples/07_final_code.py similarity index 100% rename from docs/02_introduction/code_examples/07_final_code.py rename to docs/01_introduction/code_examples/07_final_code.py diff --git a/docs/02_introduction/code_examples/07_first_code.py b/docs/01_introduction/code_examples/07_first_code.py similarity index 100% rename from docs/02_introduction/code_examples/07_first_code.py rename to docs/01_introduction/code_examples/07_first_code.py diff --git a/docs/02_introduction/code_examples/08_main.py b/docs/01_introduction/code_examples/08_main.py similarity index 100% rename from docs/02_introduction/code_examples/08_main.py rename to docs/01_introduction/code_examples/08_main.py diff --git a/docs/02_introduction/code_examples/08_routes.py b/docs/01_introduction/code_examples/08_routes.py similarity index 100% rename from docs/02_introduction/code_examples/08_routes.py rename to docs/01_introduction/code_examples/08_routes.py diff --git a/docs/02_introduction/code_examples/09_apify_sdk.py b/docs/01_introduction/code_examples/09_apify_sdk.py similarity index 100% rename from docs/02_introduction/code_examples/09_apify_sdk.py rename to docs/01_introduction/code_examples/09_apify_sdk.py diff --git a/docs/02_introduction/code_examples/__init__.py b/docs/01_introduction/code_examples/__init__.py similarity index 100% rename from docs/02_introduction/code_examples/__init__.py rename to docs/01_introduction/code_examples/__init__.py diff --git a/docs/01_quick-start/code_examples/beautifulsoup_crawler_example.py b/docs/01_introduction/code_examples/quick_start/beautifulsoup_crawler_example.py similarity index 100% rename from docs/01_quick-start/code_examples/beautifulsoup_crawler_example.py rename to docs/01_introduction/code_examples/quick_start/beautifulsoup_crawler_example.py diff --git a/docs/01_quick-start/code_examples/parsel_crawler_example.py b/docs/01_introduction/code_examples/quick_start/parsel_crawler_example.py similarity index 100% rename from docs/01_quick-start/code_examples/parsel_crawler_example.py rename to docs/01_introduction/code_examples/quick_start/parsel_crawler_example.py diff --git a/docs/01_quick-start/code_examples/playwright_crawler_example.py b/docs/01_introduction/code_examples/quick_start/playwright_crawler_example.py similarity index 100% rename from docs/01_quick-start/code_examples/playwright_crawler_example.py rename to docs/01_introduction/code_examples/quick_start/playwright_crawler_example.py diff --git a/docs/01_quick-start/code_examples/playwright_crawler_headful_example.py b/docs/01_introduction/code_examples/quick_start/playwright_crawler_headful_example.py similarity index 100% rename from docs/01_quick-start/code_examples/playwright_crawler_headful_example.py rename to docs/01_introduction/code_examples/quick_start/playwright_crawler_headful_example.py diff --git a/docs/02_introduction/code_examples/routes.py b/docs/01_introduction/code_examples/routes.py similarity index 100% rename from docs/02_introduction/code_examples/routes.py rename to docs/01_introduction/code_examples/routes.py diff --git a/docs/02_introduction/index.mdx b/docs/01_introduction/index.mdx similarity index 100% rename from docs/02_introduction/index.mdx rename to docs/01_introduction/index.mdx diff --git a/docs/01_quick-start/index.mdx b/docs/01_introduction/quick-start.mdx similarity index 91% rename from docs/01_quick-start/index.mdx rename to docs/01_introduction/quick-start.mdx index c8b61ec0eb..cbedc02ba8 100644 --- a/docs/01_quick-start/index.mdx +++ b/docs/01_introduction/quick-start.mdx @@ -1,6 +1,7 @@ --- id: quick-start title: Quick start +slug: /quick-start description: Build and run your first Crawlee crawler in a few minutes, and pick the crawler type that fits your project. --- @@ -10,13 +11,13 @@ import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_example.py'; -import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_example.py'; -import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_example.py'; +import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/quick_start/beautifulsoup_crawler_example.py'; +import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/quick_start/parsel_crawler_example.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/quick_start/playwright_crawler_example.py'; -import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/playwright_crawler_headful_example.py'; +import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/quick_start/playwright_crawler_headful_example.py'; -This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](../02_introduction/index.mdx) section, which provides a comprehensive step-by-step guide to creating your first scraper. +This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](./introduction) section, which provides a comprehensive step-by-step guide to creating your first scraper. ## Choose your crawler @@ -62,7 +63,7 @@ If you plan to use the `PlaywrightCrawler` playwright install ``` -For detailed installation instructions, see the [Setting up](../02_introduction/01_setting_up.mdx) documentation page. +For detailed installation instructions, see the [Setting up](./introduction/setting-up) documentation page. ## Crawling @@ -129,6 +130,6 @@ If you want to change the storage directory, you can set the `CRAWLEE_STORAGE_DI ## Examples and further reading -For more examples showcasing various features of Crawlee, visit the [Guides](./guides) section of the documentation. To get a deeper understanding of Crawlee and its components, read the step-by-step [Introduction](../02_introduction/index.mdx) guide. +For more examples showcasing various features of Crawlee, visit the [Guides](./guides) section of the documentation. To get a deeper understanding of Crawlee and its components, read the step-by-step [Introduction](./introduction) guide. [//]: # (TODO: add related links once they are ready) diff --git a/docs/03_concepts/01_architecture_overview.mdx b/docs/02_concepts/01_architecture_overview.mdx similarity index 99% rename from docs/03_concepts/01_architecture_overview.mdx rename to docs/02_concepts/01_architecture_overview.mdx index 21b2b7ef7c..9c7550a225 100644 --- a/docs/03_concepts/01_architecture_overview.mdx +++ b/docs/02_concepts/01_architecture_overview.mdx @@ -93,7 +93,7 @@ You can learn more about HTTP crawlers on the [HTTP crawlers](./http-crawlers) p Browser crawlers use a real browser to render pages, enabling scraping of sites that require JavaScript. They manage browser instances, pages, and context lifecycles. Crawlee provides two browser crawlers: - `PlaywrightCrawler` utilizes the [Playwright](https://playwright.dev/) library and provides a high-level API for controlling and navigating browsers. You can learn more about it on the [Playwright crawler](./playwright-crawler) page. -- `StagehandCrawler` extends `PlaywrightCrawler` with AI-powered browser automation via [Stagehand](https://github.com/browserbase/stagehand). It adds natural-language methods (`act`, `extract`, `observe`, `execute`) directly on the page object. You can learn more about it in the [Stagehand crawler guide](../guides/stagehand-crawler). +- `StagehandCrawler` extends `PlaywrightCrawler` with AI-powered browser automation via [Stagehand](https://github.com/browserbase/stagehand). It adds natural-language methods (`act`, `extract`, `observe`, `execute`) directly on the page object. You can learn more about it in the [Stagehand crawler guide](./stagehand-crawler). ### Adaptive crawler diff --git a/docs/03_concepts/05_request_router.mdx b/docs/02_concepts/04_request_router.mdx similarity index 100% rename from docs/03_concepts/05_request_router.mdx rename to docs/02_concepts/04_request_router.mdx diff --git a/docs/03_concepts/02_http_crawlers.mdx b/docs/02_concepts/05_http_crawlers.mdx similarity index 100% rename from docs/03_concepts/02_http_crawlers.mdx rename to docs/02_concepts/05_http_crawlers.mdx diff --git a/docs/03_concepts/03_playwright_crawler.mdx b/docs/02_concepts/06_playwright_crawler.mdx similarity index 100% rename from docs/03_concepts/03_playwright_crawler.mdx rename to docs/02_concepts/06_playwright_crawler.mdx diff --git a/docs/03_concepts/04_playwright_crawler_adaptive.mdx b/docs/02_concepts/07_playwright_crawler_adaptive.mdx similarity index 100% rename from docs/03_concepts/04_playwright_crawler_adaptive.mdx rename to docs/02_concepts/07_playwright_crawler_adaptive.mdx diff --git a/docs/04_guides/13_pydantic_ai_crawler.mdx b/docs/02_concepts/08_pydantic_ai_crawler.mdx similarity index 100% rename from docs/04_guides/13_pydantic_ai_crawler.mdx rename to docs/02_concepts/08_pydantic_ai_crawler.mdx diff --git a/docs/04_guides/14_stagehand_crawler.mdx b/docs/02_concepts/09_stagehand_crawler.mdx similarity index 99% rename from docs/04_guides/14_stagehand_crawler.mdx rename to docs/02_concepts/09_stagehand_crawler.mdx index 7bc7cab98b..0abbd98180 100644 --- a/docs/04_guides/14_stagehand_crawler.mdx +++ b/docs/02_concepts/09_stagehand_crawler.mdx @@ -29,7 +29,7 @@ Use `StagehandCrawler` when: - **Interactions are complex** - multi-step forms, dynamic menus, or context-dependent flows that are hard to script. - **Rapid prototyping** - you want to build a scraper quickly without spending time reverse-engineering the page structure. -For straightforward scraping tasks where the page structure is stable and well-known, `PlaywrightCrawler` is more efficient, read more on the [Playwright crawler](../concepts/playwright-crawler) page. +For straightforward scraping tasks where the page structure is stable and well-known, `PlaywrightCrawler` is more efficient, read more on the [Playwright crawler](./playwright-crawler) page. ## Installation diff --git a/docs/03_concepts/06_request_loaders.mdx b/docs/02_concepts/10_request_loaders.mdx similarity index 100% rename from docs/03_concepts/06_request_loaders.mdx rename to docs/02_concepts/10_request_loaders.mdx diff --git a/docs/03_concepts/07_storages.mdx b/docs/02_concepts/11_storages.mdx similarity index 100% rename from docs/03_concepts/07_storages.mdx rename to docs/02_concepts/11_storages.mdx diff --git a/docs/03_concepts/08_storage_clients.mdx b/docs/02_concepts/12_storage_clients.mdx similarity index 100% rename from docs/03_concepts/08_storage_clients.mdx rename to docs/02_concepts/12_storage_clients.mdx diff --git a/docs/03_concepts/09_http_clients.mdx b/docs/02_concepts/13_http_clients.mdx similarity index 100% rename from docs/03_concepts/09_http_clients.mdx rename to docs/02_concepts/13_http_clients.mdx diff --git a/docs/03_concepts/10_session_management.mdx b/docs/02_concepts/14_session_management.mdx similarity index 100% rename from docs/03_concepts/10_session_management.mdx rename to docs/02_concepts/14_session_management.mdx diff --git a/docs/03_concepts/11_cookie_management.mdx b/docs/02_concepts/15_cookie_management.mdx similarity index 100% rename from docs/03_concepts/11_cookie_management.mdx rename to docs/02_concepts/15_cookie_management.mdx diff --git a/docs/03_concepts/12_http_headers.mdx b/docs/02_concepts/16_http_headers.mdx similarity index 100% rename from docs/03_concepts/12_http_headers.mdx rename to docs/02_concepts/16_http_headers.mdx diff --git a/docs/03_concepts/13_proxy_management.mdx b/docs/02_concepts/17_proxy_management.mdx similarity index 100% rename from docs/03_concepts/13_proxy_management.mdx rename to docs/02_concepts/17_proxy_management.mdx diff --git a/docs/03_concepts/14_scaling_crawlers.mdx b/docs/02_concepts/18_scaling_crawlers.mdx similarity index 100% rename from docs/03_concepts/14_scaling_crawlers.mdx rename to docs/02_concepts/18_scaling_crawlers.mdx diff --git a/docs/03_concepts/15_request_throttling.mdx b/docs/02_concepts/19_request_throttling.mdx similarity index 100% rename from docs/03_concepts/15_request_throttling.mdx rename to docs/02_concepts/19_request_throttling.mdx diff --git a/docs/03_concepts/16_error_handling.mdx b/docs/02_concepts/20_error_handling.mdx similarity index 100% rename from docs/03_concepts/16_error_handling.mdx rename to docs/02_concepts/20_error_handling.mdx diff --git a/docs/03_concepts/17_logging.mdx b/docs/02_concepts/23_logging.mdx similarity index 100% rename from docs/03_concepts/17_logging.mdx rename to docs/02_concepts/23_logging.mdx diff --git a/docs/03_concepts/18_service_locator.mdx b/docs/02_concepts/25_service_locator.mdx similarity index 100% rename from docs/03_concepts/18_service_locator.mdx rename to docs/02_concepts/25_service_locator.mdx diff --git a/docs/04_guides/12_extending_crawlee.mdx b/docs/02_concepts/26_extending_crawlee.mdx similarity index 93% rename from docs/04_guides/12_extending_crawlee.mdx rename to docs/02_concepts/26_extending_crawlee.mdx index 07c4b7144f..e7e755f1d5 100644 --- a/docs/04_guides/12_extending_crawlee.mdx +++ b/docs/02_concepts/26_extending_crawlee.mdx @@ -124,7 +124,7 @@ The context exposes the parsed data to handlers, and the crawler ties the parser Browser crawlers use the same orchestration with a browser-backed context. Extend `PlaywrightCrawler` when an integration needs crawler-level browser behavior or a different handler context. `StagehandCrawler` is an example. It extends `PlaywrightCrawler` with a Stagehand-specific context and browser behavior. If only browser launch or lifecycle differs, a browser plugin is the narrower extension point. -See the [HTTP crawlers](../concepts/http-crawlers) page for a worked example built on `selectolax`, and the [Stagehand crawler guide](./stagehand-crawler) for `StagehandCrawler` itself. The [Architecture overview](../concepts/architecture-overview) explains how HTTP and browser crawlers relate to the other components. +See the [HTTP crawlers](./http-crawlers) page for a worked example built on `selectolax`, and the [Stagehand crawler guide](./stagehand-crawler) for `StagehandCrawler` itself. The [Architecture overview](./architecture-overview) explains how HTTP and browser crawlers relate to the other components. ### HTTP clients @@ -139,7 +139,7 @@ The contract is `HttpClient`: Crawlee ships `ImpitHttpClient`, `HttpxHttpClient`, and `CurlImpersonateHttpClient`. -See the [HTTP clients](../concepts/http-clients) page for the libraries these clients wrap and their installation requirements. +See the [HTTP clients](./http-clients) page for the libraries these clients wrap and their installation requirements. ### Storage clients @@ -153,7 +153,7 @@ The `StorageClient` contract defines A custom backend implements all four classes. -See the [Storage clients](../concepts/storage-clients) page for the built-in implementations and a custom client example. +See the [Storage clients](./storage-clients) page for the built-in implementations and a custom client example. ### Browser plugins @@ -170,7 +170,7 @@ Implement this base contract directly when the launch and lifecycle are too spec Most integrations should start with `PlaywrightBrowserPlugin`. Configure it when its launch and context options cover the required browser. Extend it when you need a custom Playwright-compatible launch path while preserving its standard lifecycle and context handling. -See the [Playwright crawler](../concepts/playwright-crawler) page for the responsibilities a subclass has to preserve, and the [Camoufox example](./avoid-blocking#using-camoufox) for a complete integration. +See the [Playwright crawler](./playwright-crawler) page for the responsibilities a subclass has to preserve, and the [Camoufox example](../guides/avoid-blocking#using-camoufox) for a complete integration. ## Choosing an extension point diff --git a/docs/03_concepts/code_examples/cookie_management/initial_cookies.py b/docs/02_concepts/code_examples/cookie_management/initial_cookies.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/initial_cookies.py rename to docs/02_concepts/code_examples/cookie_management/initial_cookies.py diff --git a/docs/03_concepts/code_examples/cookie_management/persist_cookies.py b/docs/02_concepts/code_examples/cookie_management/persist_cookies.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/persist_cookies.py rename to docs/02_concepts/code_examples/cookie_management/persist_cookies.py diff --git a/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py b/docs/02_concepts/code_examples/cookie_management/playwright_cookies.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/playwright_cookies.py rename to docs/02_concepts/code_examples/cookie_management/playwright_cookies.py diff --git a/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py b/docs/02_concepts/code_examples/cookie_management/read_write_cookies.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/read_write_cookies.py rename to docs/02_concepts/code_examples/cookie_management/read_write_cookies.py diff --git a/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py b/docs/02_concepts/code_examples/cookie_management/retry_pinned_session.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py rename to docs/02_concepts/code_examples/cookie_management/retry_pinned_session.py diff --git a/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py b/docs/02_concepts/code_examples/cookie_management/retry_restore_cookies.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py rename to docs/02_concepts/code_examples/cookie_management/retry_restore_cookies.py diff --git a/docs/03_concepts/code_examples/cookie_management/retry_single_session.py b/docs/02_concepts/code_examples/cookie_management/retry_single_session.py similarity index 100% rename from docs/03_concepts/code_examples/cookie_management/retry_single_session.py rename to docs/02_concepts/code_examples/cookie_management/retry_single_session.py diff --git a/docs/03_concepts/code_examples/error_handling/change_handle_error_status.py b/docs/02_concepts/code_examples/error_handling/change_handle_error_status.py similarity index 100% rename from docs/03_concepts/code_examples/error_handling/change_handle_error_status.py rename to docs/02_concepts/code_examples/error_handling/change_handle_error_status.py diff --git a/docs/03_concepts/code_examples/error_handling/disable_retry.py b/docs/02_concepts/code_examples/error_handling/disable_retry.py similarity index 100% rename from docs/03_concepts/code_examples/error_handling/disable_retry.py rename to docs/02_concepts/code_examples/error_handling/disable_retry.py diff --git a/docs/03_concepts/code_examples/error_handling/handle_proxy_error.py b/docs/02_concepts/code_examples/error_handling/handle_proxy_error.py similarity index 100% rename from docs/03_concepts/code_examples/error_handling/handle_proxy_error.py rename to docs/02_concepts/code_examples/error_handling/handle_proxy_error.py diff --git a/docs/03_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py b/docs/02_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py similarity index 100% rename from docs/03_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py rename to docs/02_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py diff --git a/docs/03_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py b/docs/02_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py similarity index 100% rename from docs/03_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py rename to docs/02_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py diff --git a/docs/03_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py b/docs/02_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py rename to docs/02_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py diff --git a/docs/03_concepts/code_examples/http_clients/parsel_httpx_example.py b/docs/02_concepts/code_examples/http_clients/parsel_httpx_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_clients/parsel_httpx_example.py rename to docs/02_concepts/code_examples/http_clients/parsel_httpx_example.py diff --git a/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py b/docs/02_concepts/code_examples/http_clients/parsel_impit_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_clients/parsel_impit_example.py rename to docs/02_concepts/code_examples/http_clients/parsel_impit_example.py diff --git a/docs/03_concepts/code_examples/http_crawlers/__init__.py b/docs/02_concepts/code_examples/http_crawlers/__init__.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/__init__.py rename to docs/02_concepts/code_examples/http_crawlers/__init__.py diff --git a/docs/03_concepts/code_examples/http_crawlers/beautifulsoup_example.py b/docs/02_concepts/code_examples/http_crawlers/beautifulsoup_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/beautifulsoup_example.py rename to docs/02_concepts/code_examples/http_crawlers/beautifulsoup_example.py diff --git a/docs/03_concepts/code_examples/http_crawlers/custom_crawler_example.py b/docs/02_concepts/code_examples/http_crawlers/custom_crawler_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/custom_crawler_example.py rename to docs/02_concepts/code_examples/http_crawlers/custom_crawler_example.py diff --git a/docs/03_concepts/code_examples/http_crawlers/file_download.py b/docs/02_concepts/code_examples/http_crawlers/file_download.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/file_download.py rename to docs/02_concepts/code_examples/http_crawlers/file_download.py diff --git a/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py b/docs/02_concepts/code_examples/http_crawlers/file_download_stream.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/file_download_stream.py rename to docs/02_concepts/code_examples/http_crawlers/file_download_stream.py diff --git a/docs/03_concepts/code_examples/http_crawlers/http_example.py b/docs/02_concepts/code_examples/http_crawlers/http_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/http_example.py rename to docs/02_concepts/code_examples/http_crawlers/http_example.py diff --git a/docs/03_concepts/code_examples/http_crawlers/lexbor_parser.py b/docs/02_concepts/code_examples/http_crawlers/lexbor_parser.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/lexbor_parser.py rename to docs/02_concepts/code_examples/http_crawlers/lexbor_parser.py diff --git a/docs/03_concepts/code_examples/http_crawlers/lxml_parser.py b/docs/02_concepts/code_examples/http_crawlers/lxml_parser.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/lxml_parser.py rename to docs/02_concepts/code_examples/http_crawlers/lxml_parser.py diff --git a/docs/03_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py b/docs/02_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py rename to docs/02_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py diff --git a/docs/03_concepts/code_examples/http_crawlers/parsel_example.py b/docs/02_concepts/code_examples/http_crawlers/parsel_example.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/parsel_example.py rename to docs/02_concepts/code_examples/http_crawlers/parsel_example.py diff --git a/docs/03_concepts/code_examples/http_crawlers/pyquery_parser.py b/docs/02_concepts/code_examples/http_crawlers/pyquery_parser.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/pyquery_parser.py rename to docs/02_concepts/code_examples/http_crawlers/pyquery_parser.py diff --git a/docs/03_concepts/code_examples/http_crawlers/scrapling_parser.py b/docs/02_concepts/code_examples/http_crawlers/scrapling_parser.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/scrapling_parser.py rename to docs/02_concepts/code_examples/http_crawlers/scrapling_parser.py diff --git a/docs/03_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py b/docs/02_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py rename to docs/02_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py diff --git a/docs/03_concepts/code_examples/http_crawlers/selectolax_context.py b/docs/02_concepts/code_examples/http_crawlers/selectolax_context.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/selectolax_context.py rename to docs/02_concepts/code_examples/http_crawlers/selectolax_context.py diff --git a/docs/03_concepts/code_examples/http_crawlers/selectolax_crawler.py b/docs/02_concepts/code_examples/http_crawlers/selectolax_crawler.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/selectolax_crawler.py rename to docs/02_concepts/code_examples/http_crawlers/selectolax_crawler.py diff --git a/docs/03_concepts/code_examples/http_crawlers/selectolax_crawler_run.py b/docs/02_concepts/code_examples/http_crawlers/selectolax_crawler_run.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/selectolax_crawler_run.py rename to docs/02_concepts/code_examples/http_crawlers/selectolax_crawler_run.py diff --git a/docs/03_concepts/code_examples/http_crawlers/selectolax_parser.py b/docs/02_concepts/code_examples/http_crawlers/selectolax_parser.py similarity index 100% rename from docs/03_concepts/code_examples/http_crawlers/selectolax_parser.py rename to docs/02_concepts/code_examples/http_crawlers/selectolax_parser.py diff --git a/docs/03_concepts/code_examples/http_headers/browser_page_headers.py b/docs/02_concepts/code_examples/http_headers/browser_page_headers.py similarity index 100% rename from docs/03_concepts/code_examples/http_headers/browser_page_headers.py rename to docs/02_concepts/code_examples/http_headers/browser_page_headers.py diff --git a/docs/03_concepts/code_examples/http_headers/set_headers.py b/docs/02_concepts/code_examples/http_headers/set_headers.py similarity index 100% rename from docs/03_concepts/code_examples/http_headers/set_headers.py rename to docs/02_concepts/code_examples/http_headers/set_headers.py diff --git a/docs/03_concepts/code_examples/logging/configure_json_logging.py b/docs/02_concepts/code_examples/logging/configure_json_logging.py similarity index 100% rename from docs/03_concepts/code_examples/logging/configure_json_logging.py rename to docs/02_concepts/code_examples/logging/configure_json_logging.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/browser_configuration_example.py b/docs/02_concepts/code_examples/playwright_crawler/browser_configuration_example.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/browser_configuration_example.py rename to docs/02_concepts/code_examples/playwright_crawler/browser_configuration_example.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py b/docs/02_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py rename to docs/02_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py b/docs/02_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py rename to docs/02_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py b/docs/02_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py rename to docs/02_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/multiple_launch_example.py b/docs/02_concepts/code_examples/playwright_crawler/multiple_launch_example.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/multiple_launch_example.py rename to docs/02_concepts/code_examples/playwright_crawler/multiple_launch_example.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/navigation_hooks_example.py b/docs/02_concepts/code_examples/playwright_crawler/navigation_hooks_example.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/navigation_hooks_example.py rename to docs/02_concepts/code_examples/playwright_crawler/navigation_hooks_example.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/playwright_block_requests.py b/docs/02_concepts/code_examples/playwright_crawler/playwright_block_requests.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/playwright_block_requests.py rename to docs/02_concepts/code_examples/playwright_crawler/playwright_block_requests.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/playwright_crawler.py b/docs/02_concepts/code_examples/playwright_crawler/playwright_crawler.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/playwright_crawler.py rename to docs/02_concepts/code_examples/playwright_crawler/playwright_crawler.py diff --git a/docs/03_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py b/docs/02_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py rename to docs/02_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py diff --git a/docs/03_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py b/docs/02_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py rename to docs/02_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py diff --git a/docs/03_concepts/code_examples/playwright_crawler_adaptive/handler.py b/docs/02_concepts/code_examples/playwright_crawler_adaptive/handler.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler_adaptive/handler.py rename to docs/02_concepts/code_examples/playwright_crawler_adaptive/handler.py diff --git a/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py b/docs/02_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py rename to docs/02_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py diff --git a/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py b/docs/02_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py rename to docs/02_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py diff --git a/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py b/docs/02_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py rename to docs/02_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py diff --git a/docs/03_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py b/docs/02_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py similarity index 100% rename from docs/03_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py rename to docs/02_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py diff --git a/docs/03_concepts/code_examples/proxy_management/inspecting_bs_example.py b/docs/02_concepts/code_examples/proxy_management/inspecting_bs_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/inspecting_bs_example.py rename to docs/02_concepts/code_examples/proxy_management/inspecting_bs_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/inspecting_pw_example.py b/docs/02_concepts/code_examples/proxy_management/inspecting_pw_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/inspecting_pw_example.py rename to docs/02_concepts/code_examples/proxy_management/inspecting_pw_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/integration_bs_example.py b/docs/02_concepts/code_examples/proxy_management/integration_bs_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/integration_bs_example.py rename to docs/02_concepts/code_examples/proxy_management/integration_bs_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/integration_pw_example.py b/docs/02_concepts/code_examples/proxy_management/integration_pw_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/integration_pw_example.py rename to docs/02_concepts/code_examples/proxy_management/integration_pw_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/quick_start_example.py b/docs/02_concepts/code_examples/proxy_management/quick_start_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/quick_start_example.py rename to docs/02_concepts/code_examples/proxy_management/quick_start_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/session_bs_example.py b/docs/02_concepts/code_examples/proxy_management/session_bs_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/session_bs_example.py rename to docs/02_concepts/code_examples/proxy_management/session_bs_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/session_pw_example.py b/docs/02_concepts/code_examples/proxy_management/session_pw_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/session_pw_example.py rename to docs/02_concepts/code_examples/proxy_management/session_pw_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/tiers_bs_example.py b/docs/02_concepts/code_examples/proxy_management/tiers_bs_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/tiers_bs_example.py rename to docs/02_concepts/code_examples/proxy_management/tiers_bs_example.py diff --git a/docs/03_concepts/code_examples/proxy_management/tiers_pw_example.py b/docs/02_concepts/code_examples/proxy_management/tiers_pw_example.py similarity index 100% rename from docs/03_concepts/code_examples/proxy_management/tiers_pw_example.py rename to docs/02_concepts/code_examples/proxy_management/tiers_pw_example.py diff --git a/docs/04_guides/code_examples/pydantic_ai_crawler/additional_instructions_example.py b/docs/02_concepts/code_examples/pydantic_ai_crawler/additional_instructions_example.py similarity index 100% rename from docs/04_guides/code_examples/pydantic_ai_crawler/additional_instructions_example.py rename to docs/02_concepts/code_examples/pydantic_ai_crawler/additional_instructions_example.py diff --git a/docs/04_guides/code_examples/pydantic_ai_crawler/basic_example.py b/docs/02_concepts/code_examples/pydantic_ai_crawler/basic_example.py similarity index 100% rename from docs/04_guides/code_examples/pydantic_ai_crawler/basic_example.py rename to docs/02_concepts/code_examples/pydantic_ai_crawler/basic_example.py diff --git a/docs/04_guides/code_examples/pydantic_ai_crawler/custom_distiller_example.py b/docs/02_concepts/code_examples/pydantic_ai_crawler/custom_distiller_example.py similarity index 100% rename from docs/04_guides/code_examples/pydantic_ai_crawler/custom_distiller_example.py rename to docs/02_concepts/code_examples/pydantic_ai_crawler/custom_distiller_example.py diff --git a/docs/04_guides/code_examples/pydantic_ai_crawler/debugging_example.py b/docs/02_concepts/code_examples/pydantic_ai_crawler/debugging_example.py similarity index 100% rename from docs/04_guides/code_examples/pydantic_ai_crawler/debugging_example.py rename to docs/02_concepts/code_examples/pydantic_ai_crawler/debugging_example.py diff --git a/docs/04_guides/code_examples/pydantic_ai_crawler/selector_extractor_example.py b/docs/02_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py similarity index 100% rename from docs/04_guides/code_examples/pydantic_ai_crawler/selector_extractor_example.py rename to docs/02_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py diff --git a/docs/04_guides/code_examples/pydantic_ai_crawler/usage_limit_example.py b/docs/02_concepts/code_examples/pydantic_ai_crawler/usage_limit_example.py similarity index 100% rename from docs/04_guides/code_examples/pydantic_ai_crawler/usage_limit_example.py rename to docs/02_concepts/code_examples/pydantic_ai_crawler/usage_limit_example.py diff --git a/docs/03_concepts/code_examples/request_loaders/rl_basic_example.py b/docs/02_concepts/code_examples/request_loaders/rl_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/rl_basic_example.py rename to docs/02_concepts/code_examples/request_loaders/rl_basic_example.py diff --git a/docs/03_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py b/docs/02_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py rename to docs/02_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py diff --git a/docs/03_concepts/code_examples/request_loaders/rl_tandem_example.py b/docs/02_concepts/code_examples/request_loaders/rl_tandem_example.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/rl_tandem_example.py rename to docs/02_concepts/code_examples/request_loaders/rl_tandem_example.py diff --git a/docs/03_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py b/docs/02_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py rename to docs/02_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py diff --git a/docs/03_concepts/code_examples/request_loaders/sitemap_basic_example.py b/docs/02_concepts/code_examples/request_loaders/sitemap_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/sitemap_basic_example.py rename to docs/02_concepts/code_examples/request_loaders/sitemap_basic_example.py diff --git a/docs/03_concepts/code_examples/request_loaders/sitemap_example_with_persist.py b/docs/02_concepts/code_examples/request_loaders/sitemap_example_with_persist.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/sitemap_example_with_persist.py rename to docs/02_concepts/code_examples/request_loaders/sitemap_example_with_persist.py diff --git a/docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example.py b/docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example.py rename to docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example.py diff --git a/docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py b/docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py rename to docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py diff --git a/docs/03_concepts/code_examples/request_loaders/using_sitemap_request_loader.py b/docs/02_concepts/code_examples/request_loaders/using_sitemap_request_loader.py similarity index 100% rename from docs/03_concepts/code_examples/request_loaders/using_sitemap_request_loader.py rename to docs/02_concepts/code_examples/request_loaders/using_sitemap_request_loader.py diff --git a/docs/03_concepts/code_examples/request_router/adaptive_crawler_handlers.py b/docs/02_concepts/code_examples/request_router/adaptive_crawler_handlers.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/adaptive_crawler_handlers.py rename to docs/02_concepts/code_examples/request_router/adaptive_crawler_handlers.py diff --git a/docs/03_concepts/code_examples/request_router/basic_request_handlers.py b/docs/02_concepts/code_examples/request_router/basic_request_handlers.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/basic_request_handlers.py rename to docs/02_concepts/code_examples/request_router/basic_request_handlers.py diff --git a/docs/03_concepts/code_examples/request_router/custom_router_default_only.py b/docs/02_concepts/code_examples/request_router/custom_router_default_only.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/custom_router_default_only.py rename to docs/02_concepts/code_examples/request_router/custom_router_default_only.py diff --git a/docs/03_concepts/code_examples/request_router/error_handler.py b/docs/02_concepts/code_examples/request_router/error_handler.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/error_handler.py rename to docs/02_concepts/code_examples/request_router/error_handler.py diff --git a/docs/03_concepts/code_examples/request_router/failed_request_handler.py b/docs/02_concepts/code_examples/request_router/failed_request_handler.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/failed_request_handler.py rename to docs/02_concepts/code_examples/request_router/failed_request_handler.py diff --git a/docs/03_concepts/code_examples/request_router/http_pre_navigation.py b/docs/02_concepts/code_examples/request_router/http_pre_navigation.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/http_pre_navigation.py rename to docs/02_concepts/code_examples/request_router/http_pre_navigation.py diff --git a/docs/03_concepts/code_examples/request_router/playwright_pre_navigation.py b/docs/02_concepts/code_examples/request_router/playwright_pre_navigation.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/playwright_pre_navigation.py rename to docs/02_concepts/code_examples/request_router/playwright_pre_navigation.py diff --git a/docs/03_concepts/code_examples/request_router/router_middleware.py b/docs/02_concepts/code_examples/request_router/router_middleware.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/router_middleware.py rename to docs/02_concepts/code_examples/request_router/router_middleware.py diff --git a/docs/03_concepts/code_examples/request_router/simple_default_handler.py b/docs/02_concepts/code_examples/request_router/simple_default_handler.py similarity index 100% rename from docs/03_concepts/code_examples/request_router/simple_default_handler.py rename to docs/02_concepts/code_examples/request_router/simple_default_handler.py diff --git a/docs/03_concepts/code_examples/request_throttling/throttling_example.py b/docs/02_concepts/code_examples/request_throttling/throttling_example.py similarity index 100% rename from docs/03_concepts/code_examples/request_throttling/throttling_example.py rename to docs/02_concepts/code_examples/request_throttling/throttling_example.py diff --git a/docs/03_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/02_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py similarity index 100% rename from docs/03_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py rename to docs/02_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py diff --git a/docs/03_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/02_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py similarity index 100% rename from docs/03_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py rename to docs/02_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py diff --git a/docs/03_concepts/code_examples/service_locator/service_conflicts.py b/docs/02_concepts/code_examples/service_locator/service_conflicts.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_conflicts.py rename to docs/02_concepts/code_examples/service_locator/service_conflicts.py diff --git a/docs/03_concepts/code_examples/service_locator/service_crawler_configuration.py b/docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_crawler_configuration.py rename to docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py diff --git a/docs/03_concepts/code_examples/service_locator/service_crawler_event_manager.py b/docs/02_concepts/code_examples/service_locator/service_crawler_event_manager.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_crawler_event_manager.py rename to docs/02_concepts/code_examples/service_locator/service_crawler_event_manager.py diff --git a/docs/03_concepts/code_examples/service_locator/service_crawler_storage_client.py b/docs/02_concepts/code_examples/service_locator/service_crawler_storage_client.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_crawler_storage_client.py rename to docs/02_concepts/code_examples/service_locator/service_crawler_storage_client.py diff --git a/docs/03_concepts/code_examples/service_locator/service_locator_configuration.py b/docs/02_concepts/code_examples/service_locator/service_locator_configuration.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_locator_configuration.py rename to docs/02_concepts/code_examples/service_locator/service_locator_configuration.py diff --git a/docs/03_concepts/code_examples/service_locator/service_locator_event_manager.py b/docs/02_concepts/code_examples/service_locator/service_locator_event_manager.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_locator_event_manager.py rename to docs/02_concepts/code_examples/service_locator/service_locator_event_manager.py diff --git a/docs/03_concepts/code_examples/service_locator/service_locator_storage_client.py b/docs/02_concepts/code_examples/service_locator/service_locator_storage_client.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_locator_storage_client.py rename to docs/02_concepts/code_examples/service_locator/service_locator_storage_client.py diff --git a/docs/03_concepts/code_examples/service_locator/service_storage_configuration.py b/docs/02_concepts/code_examples/service_locator/service_storage_configuration.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_storage_configuration.py rename to docs/02_concepts/code_examples/service_locator/service_storage_configuration.py diff --git a/docs/03_concepts/code_examples/service_locator/service_storage_storage_client.py b/docs/02_concepts/code_examples/service_locator/service_storage_storage_client.py similarity index 100% rename from docs/03_concepts/code_examples/service_locator/service_storage_storage_client.py rename to docs/02_concepts/code_examples/service_locator/service_storage_storage_client.py diff --git a/docs/03_concepts/code_examples/session_management/multi_sessions_http.py b/docs/02_concepts/code_examples/session_management/multi_sessions_http.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/multi_sessions_http.py rename to docs/02_concepts/code_examples/session_management/multi_sessions_http.py diff --git a/docs/03_concepts/code_examples/session_management/one_session_http.py b/docs/02_concepts/code_examples/session_management/one_session_http.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/one_session_http.py rename to docs/02_concepts/code_examples/session_management/one_session_http.py diff --git a/docs/03_concepts/code_examples/session_management/sm_basic.py b/docs/02_concepts/code_examples/session_management/sm_basic.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/sm_basic.py rename to docs/02_concepts/code_examples/session_management/sm_basic.py diff --git a/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py b/docs/02_concepts/code_examples/session_management/sm_beautifulsoup.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py rename to docs/02_concepts/code_examples/session_management/sm_beautifulsoup.py diff --git a/docs/03_concepts/code_examples/session_management/sm_http.py b/docs/02_concepts/code_examples/session_management/sm_http.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/sm_http.py rename to docs/02_concepts/code_examples/session_management/sm_http.py diff --git a/docs/03_concepts/code_examples/session_management/sm_parsel.py b/docs/02_concepts/code_examples/session_management/sm_parsel.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/sm_parsel.py rename to docs/02_concepts/code_examples/session_management/sm_parsel.py diff --git a/docs/03_concepts/code_examples/session_management/sm_playwright.py b/docs/02_concepts/code_examples/session_management/sm_playwright.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/sm_playwright.py rename to docs/02_concepts/code_examples/session_management/sm_playwright.py diff --git a/docs/03_concepts/code_examples/session_management/sm_standalone.py b/docs/02_concepts/code_examples/session_management/sm_standalone.py similarity index 100% rename from docs/03_concepts/code_examples/session_management/sm_standalone.py rename to docs/02_concepts/code_examples/session_management/sm_standalone.py diff --git a/docs/04_guides/code_examples/stagehand_crawler/basic_example.py b/docs/02_concepts/code_examples/stagehand_crawler/basic_example.py similarity index 100% rename from docs/04_guides/code_examples/stagehand_crawler/basic_example.py rename to docs/02_concepts/code_examples/stagehand_crawler/basic_example.py diff --git a/docs/04_guides/code_examples/stagehand_crawler/browserbase_example.py b/docs/02_concepts/code_examples/stagehand_crawler/browserbase_example.py similarity index 100% rename from docs/04_guides/code_examples/stagehand_crawler/browserbase_example.py rename to docs/02_concepts/code_examples/stagehand_crawler/browserbase_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py b/docs/02_concepts/code_examples/storage_clients/custom_storage_client_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py rename to docs/02_concepts/code_examples/storage_clients/custom_storage_client_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py b/docs/02_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py rename to docs/02_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py b/docs/02_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py rename to docs/02_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py b/docs/02_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py rename to docs/02_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py b/docs/02_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py rename to docs/02_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py b/docs/02_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py rename to docs/02_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/registering_storage_clients_example.py b/docs/02_concepts/code_examples/storage_clients/registering_storage_clients_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/registering_storage_clients_example.py rename to docs/02_concepts/code_examples/storage_clients/registering_storage_clients_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py b/docs/02_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py rename to docs/02_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py diff --git a/docs/03_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py b/docs/02_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py similarity index 100% rename from docs/03_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py rename to docs/02_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py diff --git a/docs/03_concepts/code_examples/storages/cleaning_do_not_purge_example.py b/docs/02_concepts/code_examples/storages/cleaning_do_not_purge_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/cleaning_do_not_purge_example.py rename to docs/02_concepts/code_examples/storages/cleaning_do_not_purge_example.py diff --git a/docs/03_concepts/code_examples/storages/cleaning_purge_explicitly_example.py b/docs/02_concepts/code_examples/storages/cleaning_purge_explicitly_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/cleaning_purge_explicitly_example.py rename to docs/02_concepts/code_examples/storages/cleaning_purge_explicitly_example.py diff --git a/docs/03_concepts/code_examples/storages/dataset_basic_example.py b/docs/02_concepts/code_examples/storages/dataset_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/dataset_basic_example.py rename to docs/02_concepts/code_examples/storages/dataset_basic_example.py diff --git a/docs/03_concepts/code_examples/storages/dataset_with_crawler_example.py b/docs/02_concepts/code_examples/storages/dataset_with_crawler_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/dataset_with_crawler_example.py rename to docs/02_concepts/code_examples/storages/dataset_with_crawler_example.py diff --git a/docs/03_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py b/docs/02_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py rename to docs/02_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py diff --git a/docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py b/docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py similarity index 100% rename from docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py rename to docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py diff --git a/docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_json.py b/docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_json.py similarity index 100% rename from docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_json.py rename to docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_json.py diff --git a/docs/03_concepts/code_examples/storages/helper_add_requests_example.py b/docs/02_concepts/code_examples/storages/helper_add_requests_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/helper_add_requests_example.py rename to docs/02_concepts/code_examples/storages/helper_add_requests_example.py diff --git a/docs/03_concepts/code_examples/storages/helper_enqueue_links_example.py b/docs/02_concepts/code_examples/storages/helper_enqueue_links_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/helper_enqueue_links_example.py rename to docs/02_concepts/code_examples/storages/helper_enqueue_links_example.py diff --git a/docs/03_concepts/code_examples/storages/kvs_basic_example.py b/docs/02_concepts/code_examples/storages/kvs_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/kvs_basic_example.py rename to docs/02_concepts/code_examples/storages/kvs_basic_example.py diff --git a/docs/03_concepts/code_examples/storages/kvs_with_crawler_example.py b/docs/02_concepts/code_examples/storages/kvs_with_crawler_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/kvs_with_crawler_example.py rename to docs/02_concepts/code_examples/storages/kvs_with_crawler_example.py diff --git a/docs/03_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py b/docs/02_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py rename to docs/02_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py diff --git a/docs/03_concepts/code_examples/storages/opening.py b/docs/02_concepts/code_examples/storages/opening.py similarity index 100% rename from docs/03_concepts/code_examples/storages/opening.py rename to docs/02_concepts/code_examples/storages/opening.py diff --git a/docs/03_concepts/code_examples/storages/rq_basic_example.py b/docs/02_concepts/code_examples/storages/rq_basic_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/rq_basic_example.py rename to docs/02_concepts/code_examples/storages/rq_basic_example.py diff --git a/docs/03_concepts/code_examples/storages/rq_with_crawler_example.py b/docs/02_concepts/code_examples/storages/rq_with_crawler_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/rq_with_crawler_example.py rename to docs/02_concepts/code_examples/storages/rq_with_crawler_example.py diff --git a/docs/03_concepts/code_examples/storages/rq_with_crawler_explicit_example.py b/docs/02_concepts/code_examples/storages/rq_with_crawler_explicit_example.py similarity index 100% rename from docs/03_concepts/code_examples/storages/rq_with_crawler_explicit_example.py rename to docs/02_concepts/code_examples/storages/rq_with_crawler_explicit_example.py diff --git a/docs/04_guides/01_crawling_links.mdx b/docs/03_guides/01_crawling_links.mdx similarity index 100% rename from docs/04_guides/01_crawling_links.mdx rename to docs/03_guides/01_crawling_links.mdx diff --git a/docs/04_guides/02_fill_and_submit_web_form.mdx b/docs/03_guides/02_fill_and_submit_web_form.mdx similarity index 100% rename from docs/04_guides/02_fill_and_submit_web_form.mdx rename to docs/03_guides/02_fill_and_submit_web_form.mdx diff --git a/docs/04_guides/03_crawler_login.mdx b/docs/03_guides/03_crawler_login.mdx similarity index 100% rename from docs/04_guides/03_crawler_login.mdx rename to docs/03_guides/03_crawler_login.mdx diff --git a/docs/04_guides/04_avoid_blocking.mdx b/docs/03_guides/04_avoid_blocking.mdx similarity index 100% rename from docs/04_guides/04_avoid_blocking.mdx rename to docs/03_guides/04_avoid_blocking.mdx diff --git a/docs/04_guides/05_respect_robots_txt_file.mdx b/docs/03_guides/05_respect_robots_txt_file.mdx similarity index 100% rename from docs/04_guides/05_respect_robots_txt_file.mdx rename to docs/03_guides/05_respect_robots_txt_file.mdx diff --git a/docs/04_guides/06_stopping_and_resuming.mdx b/docs/03_guides/06_stopping_and_resuming.mdx similarity index 100% rename from docs/04_guides/06_stopping_and_resuming.mdx rename to docs/03_guides/06_stopping_and_resuming.mdx diff --git a/docs/04_guides/07_run_parallel_crawlers.mdx b/docs/03_guides/07_run_parallel_crawlers.mdx similarity index 100% rename from docs/04_guides/07_run_parallel_crawlers.mdx rename to docs/03_guides/07_run_parallel_crawlers.mdx diff --git a/docs/04_guides/08_running_in_web_server.mdx b/docs/03_guides/08_running_in_web_server.mdx similarity index 100% rename from docs/04_guides/08_running_in_web_server.mdx rename to docs/03_guides/08_running_in_web_server.mdx diff --git a/docs/04_guides/09_trace_and_monitor_crawlers.mdx b/docs/03_guides/09_trace_and_monitor_crawlers.mdx similarity index 100% rename from docs/04_guides/09_trace_and_monitor_crawlers.mdx rename to docs/03_guides/09_trace_and_monitor_crawlers.mdx diff --git a/docs/04_guides/10_creating_web_archive.mdx b/docs/03_guides/10_creating_web_archive.mdx similarity index 100% rename from docs/04_guides/10_creating_web_archive.mdx rename to docs/03_guides/10_creating_web_archive.mdx diff --git a/docs/04_guides/11_secure_scraping.mdx b/docs/03_guides/11_secure_scraping.mdx similarity index 99% rename from docs/04_guides/11_secure_scraping.mdx rename to docs/03_guides/11_secure_scraping.mdx index 97b9b07279..599e649f54 100644 --- a/docs/04_guides/11_secure_scraping.mdx +++ b/docs/03_guides/11_secure_scraping.mdx @@ -114,7 +114,7 @@ Isolation also changes the calculus for the application-level controls. Inside a ### Running on the Apify platform -Building and maintaining isolated, egress-controlled runtimes is ongoing work. The [Apify platform](https://apify.com) runs each Actor (your crawler) in its own isolated container, without access to other users' data or to your internal network. That gives you container isolation and controlled egress out of the box. See the [Deploy on Apify](../deployment/apify-platform) guide to run your Crawlee crawler there. +Building and maintaining isolated, egress-controlled runtimes is ongoing work. The [Apify platform](https://apify.com) runs each Actor (your crawler) in its own isolated container, without access to other users' data or to your internal network. That gives you container isolation and controlled egress out of the box. See the [Deploy on Apify](./deploy-on-apify) guide to run your Crawlee crawler there. ## Conclusion diff --git a/docs/04_guides/15_scrapy_migration.mdx b/docs/03_guides/12_scrapy_migration.mdx similarity index 100% rename from docs/04_guides/15_scrapy_migration.mdx rename to docs/03_guides/12_scrapy_migration.mdx diff --git a/docs/05_deployment/apify_platform.mdx b/docs/03_guides/13_deploy_on_apify.mdx similarity index 95% rename from docs/05_deployment/apify_platform.mdx rename to docs/03_guides/13_deploy_on_apify.mdx index 88db05ca7d..6d4f952e77 100644 --- a/docs/05_deployment/apify_platform.mdx +++ b/docs/03_guides/13_deploy_on_apify.mdx @@ -1,17 +1,17 @@ --- -id: apify-platform -title: Apify platform -description: Apify platform - large-scale and high-performance web scraping +id: deploy-on-apify +title: Deploying to Apify +description: How to run your Crawlee project as an Actor on the Apify platform, with cloud storages, Apify Proxy, and platform events. --- import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import LogWithConfigExample from '!!raw-loader!./code_examples/apify/log_with_config_example.py'; -import CrawlerAsActorExample from '!!raw-loader!./code_examples/apify/crawler_as_actor_example.py'; -import ProxyExample from '!!raw-loader!./code_examples/apify/proxy_example.py'; -import ProxyAdvancedExample from '!!raw-loader!./code_examples/apify/proxy_advanced_example.py'; +import LogWithConfigExample from '!!raw-loader!./code_examples/deployment/apify/log_with_config_example.py'; +import CrawlerAsActorExample from '!!raw-loader!./code_examples/deployment/apify/crawler_as_actor_example.py'; +import ProxyExample from '!!raw-loader!./code_examples/deployment/apify/proxy_example.py'; +import ProxyAdvancedExample from '!!raw-loader!./code_examples/deployment/apify/proxy_advanced_example.py'; Apify is a [platform](https://apify.com) built to serve large-scale and high-performance web scraping and automation needs. It provides easy access to [compute instances (Actors)](#what-is-an-actor), convenient request and result storages, [proxies](../concepts/proxy-management), scheduling, webhooks, and [more in the Apify documentation](https://docs.apify.com/), accessible through a [web interface](https://console.apify.com) or an [API](https://docs.apify.com/api). @@ -127,7 +127,7 @@ Your script will be uploaded to and built on the Apify platform so that it can b ## Usage on Apify platform -You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Guides](../guides) section. +You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Guides](./.) section. ## Storages diff --git a/docs/05_deployment/aws_lambda.mdx b/docs/03_guides/14_deploy_on_aws_lambda.mdx similarity index 97% rename from docs/05_deployment/aws_lambda.mdx rename to docs/03_guides/14_deploy_on_aws_lambda.mdx index 8d7d94123a..9b71c73c4a 100644 --- a/docs/05_deployment/aws_lambda.mdx +++ b/docs/03_guides/14_deploy_on_aws_lambda.mdx @@ -1,6 +1,6 @@ --- -id: aws-lambda -title: Deploy on AWS Lambda +id: deploy-on-aws-lambda +title: Deploying to AWS Lambda description: Prepare your crawler to run on AWS Lambda. --- @@ -8,9 +8,9 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import BeautifulSoupCrawlerLambda from '!!raw-loader!./code_examples/aws/beautifulsoup_crawler_lambda.py'; -import PlaywrightCrawlerLambda from '!!raw-loader!./code_examples/aws/playwright_crawler_lambda.py'; -import PlaywrightCrawlerDockerfile from '!!raw-loader!./code_examples/aws/playwright_dockerfile'; +import BeautifulSoupCrawlerLambda from '!!raw-loader!./code_examples/deployment/aws/beautifulsoup_crawler_lambda.py'; +import PlaywrightCrawlerLambda from '!!raw-loader!./code_examples/deployment/aws/playwright_crawler_lambda.py'; +import PlaywrightCrawlerDockerfile from '!!raw-loader!./code_examples/deployment/aws/playwright_dockerfile'; [AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) is a serverless compute service that lets you run code without provisioning or managing servers. This guide covers deploying `BeautifulSoupCrawler` and `PlaywrightCrawler`. diff --git a/docs/05_deployment/google_cloud.mdx b/docs/03_guides/15_deploy_on_gcp_cloud_run_functions.mdx similarity index 92% rename from docs/05_deployment/google_cloud.mdx rename to docs/03_guides/15_deploy_on_gcp_cloud_run_functions.mdx index 5c4a3535cf..aad575ce06 100644 --- a/docs/05_deployment/google_cloud.mdx +++ b/docs/03_guides/15_deploy_on_gcp_cloud_run_functions.mdx @@ -1,6 +1,6 @@ --- -id: gcp-cloud-run-functions -title: Cloud Run functions +id: deploy-on-gcp-cloud-run-functions +title: Deploying to Google Cloud Run functions description: Prepare your crawler to run in Cloud Run functions on Google Cloud Platform. --- @@ -8,7 +8,7 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import GoogleFunctions from '!!raw-loader!./code_examples/google/google_example.py'; +import GoogleFunctions from '!!raw-loader!./code_examples/deployment/google/google_example.py'; [Google Cloud Run Functions](https://cloud.google.com/functions) is a serverless execution environment for running simple HTTP-based web scrapers. This service is best suited for lightweight crawlers that don't require browser rendering capabilities and can be executed via HTTP requests. diff --git a/docs/05_deployment/google_cloud_run.mdx b/docs/03_guides/16_deploy_on_gcp_cloud_run.mdx similarity index 91% rename from docs/05_deployment/google_cloud_run.mdx rename to docs/03_guides/16_deploy_on_gcp_cloud_run.mdx index b0e206b03c..dcb2f17688 100644 --- a/docs/05_deployment/google_cloud_run.mdx +++ b/docs/03_guides/16_deploy_on_gcp_cloud_run.mdx @@ -1,6 +1,6 @@ --- -id: gcp-cloud-run -title: Cloud Run +id: deploy-on-gcp-cloud-run +title: Deploying to Google Cloud Run description: Prepare your crawler to run in Cloud Run on Google Cloud Platform. --- @@ -8,10 +8,10 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import GoogleCloudRun from '!!raw-loader!./code_examples/google/cloud_run_example.py'; +import GoogleCloudRun from '!!raw-loader!./code_examples/deployment/google/cloud_run_example.py'; -[Google Cloud Run](https://cloud.google.com/run) is a container-based serverless platform that allows you to run web crawlers with headless browsers. This service is recommended when your Crawlee applications need browser rendering capabilities, require more granular control, or have complex dependencies that aren't supported by [Cloud Functions](./gcp-cloud-run-functions). +[Google Cloud Run](https://cloud.google.com/run) is a container-based serverless platform that allows you to run web crawlers with headless browsers. This service is recommended when your Crawlee applications need browser rendering capabilities, require more granular control, or have complex dependencies that aren't supported by [Cloud Functions](./deploy-on-gcp-cloud-run-functions). GCP Cloud Run allows you to deploy using Docker containers, giving you full control over your environment and the flexibility to use any web server framework of your choice, unlike Cloud Functions which are limited to [Flask](https://flask.palletsprojects.com/en/stable/). diff --git a/docs/04_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py b/docs/03_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py similarity index 100% rename from docs/04_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py rename to docs/03_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py diff --git a/docs/04_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py b/docs/03_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py similarity index 100% rename from docs/04_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py rename to docs/03_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py diff --git a/docs/04_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py b/docs/03_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py similarity index 100% rename from docs/04_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py rename to docs/03_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py diff --git a/docs/04_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py b/docs/03_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py similarity index 100% rename from docs/04_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py rename to docs/03_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py b/docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py rename to docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py b/docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py rename to docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py b/docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py rename to docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py b/docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py rename to docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py b/docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py rename to docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py b/docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py rename to docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py b/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py rename to docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py b/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py rename to docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py b/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py rename to docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py diff --git a/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py b/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py rename to docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py diff --git a/docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py b/docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py rename to docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py diff --git a/docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py b/docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py similarity index 100% rename from docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py rename to docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py diff --git a/docs/04_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py b/docs/03_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py similarity index 100% rename from docs/04_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py rename to docs/03_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py diff --git a/docs/04_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py b/docs/03_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py similarity index 100% rename from docs/04_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py rename to docs/03_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py diff --git a/docs/04_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py b/docs/03_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py similarity index 100% rename from docs/04_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py rename to docs/03_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py diff --git a/docs/05_deployment/code_examples/apify/crawler_as_actor_example.py b/docs/03_guides/code_examples/deployment/apify/crawler_as_actor_example.py similarity index 100% rename from docs/05_deployment/code_examples/apify/crawler_as_actor_example.py rename to docs/03_guides/code_examples/deployment/apify/crawler_as_actor_example.py diff --git a/docs/05_deployment/code_examples/apify/get_public_url.py b/docs/03_guides/code_examples/deployment/apify/get_public_url.py similarity index 100% rename from docs/05_deployment/code_examples/apify/get_public_url.py rename to docs/03_guides/code_examples/deployment/apify/get_public_url.py diff --git a/docs/05_deployment/code_examples/apify/log_with_config_example.py b/docs/03_guides/code_examples/deployment/apify/log_with_config_example.py similarity index 100% rename from docs/05_deployment/code_examples/apify/log_with_config_example.py rename to docs/03_guides/code_examples/deployment/apify/log_with_config_example.py diff --git a/docs/05_deployment/code_examples/apify/proxy_advanced_example.py b/docs/03_guides/code_examples/deployment/apify/proxy_advanced_example.py similarity index 100% rename from docs/05_deployment/code_examples/apify/proxy_advanced_example.py rename to docs/03_guides/code_examples/deployment/apify/proxy_advanced_example.py diff --git a/docs/05_deployment/code_examples/apify/proxy_example.py b/docs/03_guides/code_examples/deployment/apify/proxy_example.py similarity index 100% rename from docs/05_deployment/code_examples/apify/proxy_example.py rename to docs/03_guides/code_examples/deployment/apify/proxy_example.py diff --git a/docs/05_deployment/code_examples/aws/beautifulsoup_crawler_lambda.py b/docs/03_guides/code_examples/deployment/aws/beautifulsoup_crawler_lambda.py similarity index 100% rename from docs/05_deployment/code_examples/aws/beautifulsoup_crawler_lambda.py rename to docs/03_guides/code_examples/deployment/aws/beautifulsoup_crawler_lambda.py diff --git a/docs/05_deployment/code_examples/aws/playwright_crawler_lambda.py b/docs/03_guides/code_examples/deployment/aws/playwright_crawler_lambda.py similarity index 100% rename from docs/05_deployment/code_examples/aws/playwright_crawler_lambda.py rename to docs/03_guides/code_examples/deployment/aws/playwright_crawler_lambda.py diff --git a/docs/05_deployment/code_examples/aws/playwright_dockerfile b/docs/03_guides/code_examples/deployment/aws/playwright_dockerfile similarity index 100% rename from docs/05_deployment/code_examples/aws/playwright_dockerfile rename to docs/03_guides/code_examples/deployment/aws/playwright_dockerfile diff --git a/docs/05_deployment/code_examples/google/cloud_run_example.py b/docs/03_guides/code_examples/deployment/google/cloud_run_example.py similarity index 100% rename from docs/05_deployment/code_examples/google/cloud_run_example.py rename to docs/03_guides/code_examples/deployment/google/cloud_run_example.py diff --git a/docs/05_deployment/code_examples/google/google_example.py b/docs/03_guides/code_examples/deployment/google/google_example.py similarity index 100% rename from docs/05_deployment/code_examples/google/google_example.py rename to docs/03_guides/code_examples/deployment/google/google_example.py diff --git a/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py b/docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py similarity index 100% rename from docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py rename to docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py diff --git a/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py b/docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py similarity index 100% rename from docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py rename to docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py diff --git a/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py b/docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py similarity index 100% rename from docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py rename to docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py diff --git a/docs/04_guides/code_examples/login_crawler/http_login.py b/docs/03_guides/code_examples/login_crawler/http_login.py similarity index 100% rename from docs/04_guides/code_examples/login_crawler/http_login.py rename to docs/03_guides/code_examples/login_crawler/http_login.py diff --git a/docs/04_guides/code_examples/login_crawler/playwright_login.py b/docs/03_guides/code_examples/login_crawler/playwright_login.py similarity index 100% rename from docs/04_guides/code_examples/login_crawler/playwright_login.py rename to docs/03_guides/code_examples/login_crawler/playwright_login.py diff --git a/docs/04_guides/code_examples/login_crawler/using_browser_profiles_chrome.py b/docs/03_guides/code_examples/login_crawler/using_browser_profiles_chrome.py similarity index 100% rename from docs/04_guides/code_examples/login_crawler/using_browser_profiles_chrome.py rename to docs/03_guides/code_examples/login_crawler/using_browser_profiles_chrome.py diff --git a/docs/04_guides/code_examples/login_crawler/using_browser_profiles_firefox.py b/docs/03_guides/code_examples/login_crawler/using_browser_profiles_firefox.py similarity index 100% rename from docs/04_guides/code_examples/login_crawler/using_browser_profiles_firefox.py rename to docs/03_guides/code_examples/login_crawler/using_browser_profiles_firefox.py diff --git a/docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py b/docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py similarity index 100% rename from docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py rename to docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py diff --git a/docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py b/docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py similarity index 100% rename from docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py rename to docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py diff --git a/docs/04_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py b/docs/03_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py similarity index 100% rename from docs/04_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py rename to docs/03_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py diff --git a/docs/04_guides/code_examples/running_in_web_server/__init__.py b/docs/03_guides/code_examples/running_in_web_server/__init__.py similarity index 100% rename from docs/04_guides/code_examples/running_in_web_server/__init__.py rename to docs/03_guides/code_examples/running_in_web_server/__init__.py diff --git a/docs/04_guides/code_examples/running_in_web_server/crawler.py b/docs/03_guides/code_examples/running_in_web_server/crawler.py similarity index 100% rename from docs/04_guides/code_examples/running_in_web_server/crawler.py rename to docs/03_guides/code_examples/running_in_web_server/crawler.py diff --git a/docs/04_guides/code_examples/running_in_web_server/server.py b/docs/03_guides/code_examples/running_in_web_server/server.py similarity index 100% rename from docs/04_guides/code_examples/running_in_web_server/server.py rename to docs/03_guides/code_examples/running_in_web_server/server.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_concurrency.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_concurrency.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_concurrency.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_concurrency.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_crawlspider.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_crawlspider.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_crawlspider.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_crawlspider.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_error_handling.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_error_handling.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_error_handling.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_error_handling.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_export.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_export.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_export.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_export.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_labels.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_labels.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_labels.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_labels.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_playwright.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_playwright.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_playwright.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_playwright.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_post.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_post.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_post.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_proxy.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_proxy.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_proxy.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_proxy.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_quotes.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_quotes.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_quotes.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_quotes.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_throttling.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_throttling.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_throttling.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_throttling.py diff --git a/docs/04_guides/code_examples/scrapy_migration/crawlee_user_data.py b/docs/03_guides/code_examples/scrapy_migration/crawlee_user_data.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/crawlee_user_data.py rename to docs/03_guides/code_examples/scrapy_migration/crawlee_user_data.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_authors.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_authors.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_authors.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_authors.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_concurrency.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_concurrency.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_concurrency.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_concurrency.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_crawlspider.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_crawlspider.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_crawlspider.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_crawlspider.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_errback.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_errback.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_errback.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_errback.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_export.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_export.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_export.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_export.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_formrequest.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_formrequest.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_formrequest.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_formrequest.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_playwright.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_playwright.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_playwright.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_playwright.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_proxy.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_proxy.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_proxy.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_proxy.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_quotes.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_quotes.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_quotes.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_quotes.py diff --git a/docs/04_guides/code_examples/scrapy_migration/scrapy_throttling.py b/docs/03_guides/code_examples/scrapy_migration/scrapy_throttling.py similarity index 100% rename from docs/04_guides/code_examples/scrapy_migration/scrapy_throttling.py rename to docs/03_guides/code_examples/scrapy_migration/scrapy_throttling.py diff --git a/docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py b/docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py similarity index 100% rename from docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py rename to docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py diff --git a/docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py b/docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py similarity index 100% rename from docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py rename to docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py diff --git a/docs/04_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py b/docs/03_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py similarity index 100% rename from docs/04_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py rename to docs/03_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py diff --git a/docs/04_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py b/docs/03_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py similarity index 100% rename from docs/04_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py rename to docs/03_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py diff --git a/docs/06_upgrading/upgrading_to_v0x.md b/docs/04_upgrading/upgrading_to_v0x.md similarity index 100% rename from docs/06_upgrading/upgrading_to_v0x.md rename to docs/04_upgrading/upgrading_to_v0x.md diff --git a/docs/06_upgrading/upgrading_to_v1.md b/docs/04_upgrading/upgrading_to_v1.md similarity index 100% rename from docs/06_upgrading/upgrading_to_v1.md rename to docs/04_upgrading/upgrading_to_v1.md diff --git a/pyproject.toml b/pyproject.toml index 892ce7ea67..4d9ae7e2b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,16 +237,16 @@ indent-style = "space" "N999", # Invalid module name "T201", # `print` found ] -"**/docs/03_concepts/code_examples/error_handling/*crawler_with_error_snapshotter.py" = [ +"**/docs/02_concepts/code_examples/error_handling/*crawler_with_error_snapshotter.py" = [ "PLR2004", # Magic value used in comparison. Ignored for simplicity and readability of example code. ] -"**/docs/04_guides/code_examples/running_in_web_server/server.py" = [ +"**/docs/03_guides/code_examples/running_in_web_server/server.py" = [ "TC002", # ruff false positive. Import actually needed during runtime. ] -"**/docs/04_guides/code_examples/creating_web_archive/*.*" = [ +"**/docs/03_guides/code_examples/creating_web_archive/*.*" = [ "ASYNC230", # Ignore for simplicity of the example. ] -"**/docs/04_guides/code_examples/scrapy_migration/scrapy_*.py" = [ +"**/docs/03_guides/code_examples/scrapy_migration/scrapy_*.py" = [ "ANN", # Idiomatic Scrapy tutorial snippets are untyped; shown for comparison, not executed. "RUF012", # Scrapy declares `start_urls` as a plain class-level list. ] @@ -295,7 +295,7 @@ python-version = "3.10" include = ["src", "tests", "scripts", "docs", "website"] exclude = [ "src/crawlee/project_template", - "docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py", + "docs/02_concepts/code_examples/storage_clients/custom_storage_client_example.py", "website/versioned_docs", ] diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 9fff82d8df..24d45460ca 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -167,8 +167,11 @@ module.exports = { 'guides/error-handling', 'examples/capturing-page-snapshots-with-error-snapshotter', ], + 'concepts/pydantic-ai-crawler': ['guides/pydantic-ai-crawler'], + 'concepts/stagehand-crawler': ['guides/stagehand-crawler'], 'concepts/logging': ['examples/configure-json-logging'], 'concepts/service-locator': ['guides/service-locator'], + 'concepts/extending-crawlee': ['guides/extending-crawlee'], 'guides/crawling-links': [ 'examples/crawl-all-links-on-website', 'examples/crawl-multiple-urls', @@ -188,13 +191,20 @@ module.exports = { 'examples/playwright-crawler-with-fingerprint-generator', ], 'guides/logging-in-with-a-crawler': ['examples/using_browser_profile'], - guides: ['examples'], + 'guides/deploy-on-apify': ['deployment/apify-platform'], + 'guides/deploy-on-aws-lambda': ['deployment/aws-lambda'], + 'guides/deploy-on-gcp-cloud-run-functions': ['deployment/gcp-cloud-run-functions'], + 'guides/deploy-on-gcp-cloud-run': ['deployment/gcp-cloud-run'], + guides: ['examples', 'deployment'], }; - // Doc versions that still have the pre-restructure layout. Their routes - // (e.g. /docs/1.9/examples/...) must not get redirects generated over them. - const OLD_STRUCTURE_VERSIONS = ['0.6', '1.9']; + // Doc versions that still have the pre-restructure layout (no numbered concepts + // directory). Their routes (e.g. /docs/1.11/examples/...) must not get redirects + // generated over them. const versions = require('./versions.json'); + const hasOldStructureOnDisk = (version) => + !fs.existsSync(path.join(__dirname, 'versioned_docs', `version-${version}`, '02_concepts')); + const OLD_STRUCTURE_VERSIONS = versions.filter(hasOldStructureOnDisk); const latestHasOldStructure = OLD_STRUCTURE_VERSIONS.includes(versions[0]); const marker = '/docs/'; @@ -356,7 +366,7 @@ module.exports = { items: [ { type: 'doc', - docId: 'quick-start/quick-start', + docId: 'introduction/introduction', label: 'Docs', position: 'left', }, diff --git a/website/sidebars.js b/website/sidebars.js index cab87b4881..fae4d63e66 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -1,13 +1,24 @@ module.exports = { docs: [ - 'quick-start/quick-start', + { + type: 'doc', + id: 'introduction/introduction', + }, + { + type: 'doc', + id: 'introduction/quick-start', + }, { type: 'category', - label: 'Introduction', + label: 'Tutorial', collapsed: true, link: { - type: 'doc', - id: 'introduction/introduction', + type: 'generated-index', + title: 'Tutorial', + description: + 'A step-by-step tutorial that takes you from your first crawler to a production-ready scraper for a real website.', + slug: '/introduction/tutorial', + keywords: ['tutorial'], }, items: [ 'introduction/setting-up', @@ -29,14 +40,14 @@ module.exports = { type: 'generated-index', title: 'Concepts', description: - 'Learn how the core components of Crawlee work - crawlers, request routing, storages, sessions, and more.', + 'How the building blocks of Crawlee work: crawlers, requests, routing, storages, sessions, and more.', slug: '/concepts', keywords: ['concepts'], }, items: [ { type: 'autogenerated', - dirName: '03_concepts', + dirName: '02_concepts', }, ], }, @@ -48,42 +59,14 @@ module.exports = { type: 'generated-index', title: 'Guides', description: - 'Task-oriented guides for common crawling and scraping scenarios, integrations, and migrations.', + 'Step-by-step guides for specific crawling and scraping tasks, integrations, migrations, and deployment.', slug: '/guides', keywords: ['guides'], }, items: [ { type: 'autogenerated', - dirName: '04_guides', - }, - ], - }, - { - type: 'category', - label: 'Deployment', - collapsed: true, - link: { - type: 'generated-index', - title: 'Deployment guides', - description: 'Here you can find guides on how to deploy your crawlers to various cloud providers.', - slug: '/deployment', - }, - items: [ - { - type: 'doc', - id: 'deployment/apify-platform', - label: 'Deploy on Apify', - }, - { - type: 'doc', - id: 'deployment/aws-lambda', - label: 'Deploy on AWS Lambda', - }, - { - type: 'category', - label: 'Deploy to Google Cloud', - items: ['deployment/gcp-cloud-run-functions', 'deployment/gcp-cloud-run'], + dirName: '03_guides', }, ], }, @@ -100,7 +83,7 @@ module.exports = { items: [ { type: 'autogenerated', - dirName: '06_upgrading', + dirName: '04_upgrading', }, ], }, From b841d30ec8ab200141e99a4507174c7322694035 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 12:30:56 +0200 Subject: [PATCH 03/11] docs: add an overview page and tighten the quick start and tutorial --- docs/01_introduction/01_setting_up.mdx | 2 +- docs/01_introduction/08_refactoring.mdx | 2 +- docs/01_introduction/09_running_in_cloud.mdx | 6 +- docs/01_introduction/index.mdx | 71 +++++++++++--------- docs/01_introduction/quick-start.mdx | 40 +++++------ 5 files changed, 63 insertions(+), 58 deletions(-) diff --git a/docs/01_introduction/01_setting_up.mdx b/docs/01_introduction/01_setting_up.mdx index 75858d517c..259eb18981 100644 --- a/docs/01_introduction/01_setting_up.mdx +++ b/docs/01_introduction/01_setting_up.mdx @@ -9,7 +9,7 @@ import CodeBlock from '@theme/CodeBlock'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -This guide will help you get started with Crawlee by setting it up on your computer. Follow the steps below to ensure a smooth installation process. +This tutorial takes you from the simplest crawler, which prints a page title, to a scraper that crawls a real website and extracts product data. The first step is setting up Crawlee on your computer. ## Prerequisites diff --git a/docs/01_introduction/08_refactoring.mdx b/docs/01_introduction/08_refactoring.mdx index 8118ae9e06..e5d7597a88 100644 --- a/docs/01_introduction/08_refactoring.mdx +++ b/docs/01_introduction/08_refactoring.mdx @@ -22,7 +22,7 @@ You might be wondering about the **anti-blocking, bot-protection avoiding stealt However, the default configuration, while powerful, may not cover every scenario. -If you want to learn more, browse the [Avoid getting blocked](../guides/avoid-blocking), [Proxy management](../concepts/proxy-management) and [Session management](../concepts/session-management) guides. +To learn more, see [Avoid getting blocked](../guides/avoid-blocking), [Proxy management](../concepts/proxy-management), and [Session management](../concepts/session-management). */} To promote good coding practices, let's look at how you can use a `Router` class to better structure your crawler code. diff --git a/docs/01_introduction/09_running_in_cloud.mdx b/docs/01_introduction/09_running_in_cloud.mdx index e3523a4758..d2339a921e 100644 --- a/docs/01_introduction/09_running_in_cloud.mdx +++ b/docs/01_introduction/09_running_in_cloud.mdx @@ -12,13 +12,13 @@ import MainExample from '!!raw-loader!./code_examples/09_apify_sdk.py'; Crawlee is developed by [**Apify**](https://apify.com), the web scraping and automation platform. You could say it is the **home of Crawlee projects**. In this section you'll see how to deploy the crawler there with just a few simple steps. You can deploy a **Crawlee** project wherever you want, but using the [**Apify platform**](https://console.apify.com) will give you the best experience. -{/*In case you want to deploy your Crawlee project to other platforms, check out the [**Deployment**](../guides) section.*/} +To deploy your project to other platforms, see [Deploying to AWS Lambda](../guides/deploy-on-aws-lambda) and [Deploying to Google Cloud Run](../guides/deploy-on-gcp-cloud-run). With a few simple steps, you can convert your Crawlee project into a so-called **Actor**. Actors are serverless micro-apps that are easy to develop, run, share, and integrate. The infra, proxies, and storages are ready to go. [Learn more about Actors](https://apify.com/actors). {/*:::info Choosing between Crawlee CLI and Apify CLI for project setup -We started this guide by using the Crawlee CLI to bootstrap the project - it offers the basic Crawlee templates, including a ready-made `Dockerfile`. If you know you will be deploying your project to the Apify platform, you might want to start with the Apify CLI instead. It also offers several project templates, and those are all set up to be used on the Apify platform right ahead. +We started this tutorial by using the Crawlee CLI to bootstrap the project - it offers the basic Crawlee templates, including a ready-made `Dockerfile`. If you know you will be deploying your project to the Apify platform, you might want to start with the Apify CLI instead. It also offers several project templates, and those are all set up to be used on the Apify platform right ahead. :::*/} @@ -98,4 +98,4 @@ If you want to learn more about web scraping and browser automation, check out t ## Thank you! ๐ŸŽ‰ -That's it! Thanks for reading the whole introduction and if there's anything wrong, please ๐Ÿ™ let us know on [GitHub](https://github.com/apify/crawlee-python) or in our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! ๐Ÿ‘‹ +That's it! Thanks for reading the whole tutorial and if there's anything wrong, please ๐Ÿ™ let us know on [GitHub](https://github.com/apify/crawlee-python) or in our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! ๐Ÿ‘‹ diff --git a/docs/01_introduction/index.mdx b/docs/01_introduction/index.mdx index fc7d3dab86..7d79536567 100644 --- a/docs/01_introduction/index.mdx +++ b/docs/01_introduction/index.mdx @@ -1,56 +1,61 @@ --- id: introduction -title: Introduction -description: A step-by-step tutorial that takes you from your first crawler to a production-ready scraper for a real website. +title: Overview +description: Crawlee for Python is a web scraping and browser automation library with a unified interface for HTTP and headless browser crawling. --- import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -Crawlee covers your crawling and scraping end-to-end and helps you **build reliable scrapers. Fast.** +import OverviewExample from '!!raw-loader!roa-loader!./code_examples/quick_start/beautifulsoup_crawler_example.py'; -Your crawlers will appear human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it. +Crawlee for Python is a web scraping and browser automation library. It handles the crawling infrastructure for you: it queues and deduplicates requests, retries failures, scales concurrency to the available system resources, rotates proxies and sessions, and stores the results. You write the code that extracts the data. -## What you will learn +The same interface works for plain HTTP crawling and for headless browsers, so you can switch between them without rewriting your handlers. With the default configuration, the HTTP client impersonates a real browser and browser crawlers use generated fingerprints, which helps them get past common bot protections. -The goal of the introduction is to provide a step-by-step guide to the most important features of Crawlee. It will walk you through creating the simplest of crawlers that only prints text to console, all the way up to a full-featured scraper that collects links from a website and extracts data. + + {OverviewExample} + -## ๐Ÿ›  Features +## Features -Why Crawlee is the preferred choice for web scraping and crawling? +- One interface for HTTP and headless browser crawling. +- Automatic parallel crawling based on available CPU and memory. For details, see [Scaling crawlers](./concepts/scaling-crawlers). +- Automatic retries on errors and when the crawler gets blocked. For details, see [Error handling](./concepts/error-handling). +- Proxy rotation and session management. For details, see [Proxy management](./concepts/proxy-management) and [Session management](./concepts/session-management). +- Request routing that sends each URL to the right handler. For details, see [Request router](./concepts/request-router). +- A persistent request queue and pluggable storages for tabular data and files. For details, see [Storages](./concepts/storages). +- Full type hint coverage, built on standard `asyncio`. -### Why use Crawlee instead of just a random HTTP library with an HTML parser? +## Crawler types -- Unified interface for **HTTP & headless browser** crawling. -- Automatic **parallel crawling** based on available system resources. -- Written in Python with **type hints** - enhances DX (IDE autocompletion) and reduces bugs (static type checking). -- Automatic **retries** on errors or when you are getting blocked. -- Integrated **proxy rotation** and session management. -- Configurable **request routing** - direct URLs to the appropriate handlers. -- Persistent **queue for URLs** to crawl. -- Pluggable **storage** of both tabular data and files. -- Robust **error handling**. +Crawlee provides several crawlers that share the same base class, `BasicCrawler`: -### Why to use Crawlee rather than Scrapy? +- `BeautifulSoupCrawler` and `ParselCrawler` fetch pages over HTTP and parse the HTML. They're fast and light, but they can't run JavaScript. For details, see [HTTP crawlers](./concepts/http-crawlers). +- `PlaywrightCrawler` controls a headless browser through [Playwright](https://playwright.dev/), for pages that render their content with JavaScript. For details, see [Playwright crawler](./concepts/playwright-crawler). +- `AdaptivePlaywrightCrawler` decides per request whether a plain HTTP fetch is enough or a browser is needed. For details, see [Adaptive Playwright crawler](./concepts/adaptive-playwright-crawler). +- `PydanticAiCrawler` and `StagehandCrawler` use an LLM to extract data or drive the browser. For details, see [Pydantic AI crawler](./concepts/pydantic-ai-crawler) and [Stagehand crawler](./concepts/stagehand-crawler). -- Crawlee has out-of-the-box support for **headless browser** crawling (Playwright). -- Crawlee has a **minimalistic & elegant interface** - Set up your scraper with fewer than 10 lines of code. -- Complete **type hint** coverage. -- Based on standard **Asyncio**. +## Installation -{/* TODO: +Crawlee requires Python 3.10 or higher. It's available as the [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. The core package is small, and each crawler type comes with an optional extra. To install Crawlee with all extras, run: -### ๐Ÿ‘พ HTTP crawling +```sh +pip install 'crawlee[all]' +``` -- ... -*/} +To use the `PlaywrightCrawler`, also install the browser binaries: -{/* TODO: +```sh +playwright install +``` -### ๐Ÿ’ป Real browser crawling - -- ... -*/} +For the list of extras and for bootstrapping a project from a template with the Crawlee CLI, see [Setting up](./introduction/setting-up). ## Next steps -Next, you will install Crawlee and learn how to bootstrap projects with the prepared Crawlee templates. +- To run your first crawler in a few minutes, see the [Quick start](./quick-start). +- To build a complete scraper for a real website step by step, follow the [Tutorial](./introduction/tutorial). +- To learn how the parts of Crawlee work, read the [Concepts](./concepts). +- For solutions to specific tasks, such as logging in, filling in forms, or deploying a crawler, see the [Guides](./guides). +- If you're coming from Scrapy, see [Migrating from Scrapy](./guides/scrapy-migration). diff --git a/docs/01_introduction/quick-start.mdx b/docs/01_introduction/quick-start.mdx index cbedc02ba8..25343b0023 100644 --- a/docs/01_introduction/quick-start.mdx +++ b/docs/01_introduction/quick-start.mdx @@ -17,11 +17,11 @@ import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/qu import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/quick_start/playwright_crawler_headful_example.py'; -This short tutorial will help you start scraping with Crawlee in just a minute or two. For an in-depth understanding of how Crawlee works, check out the [Introduction](./introduction) section, which provides a comprehensive step-by-step guide to creating your first scraper. +This page gets you from installation to a running crawler in a few minutes. To build a complete scraper step by step, follow the [Tutorial](./introduction/tutorial). ## Choose your crawler -Crawlee offers the following main crawler classes: `BeautifulSoupCrawler`, `ParselCrawler`, and `PlaywrightCrawler`. All crawlers share the same interface, providing maximum flexibility when switching between them. +Crawlee's main crawler classes are `BeautifulSoupCrawler`, `ParselCrawler`, and `PlaywrightCrawler`. All crawlers share the same interface, so you can switch between them without rewriting your handlers. :::caution Minimum Python version @@ -31,21 +31,21 @@ Crawlee requires Python 3.10 or higher. ### BeautifulSoupCrawler -The `BeautifulSoupCrawler` is a plain HTTP crawler that parses HTML using the well-known [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library. It crawls the web using an HTTP client that mimics a browser. This crawler is very fast and efficient but cannot handle JavaScript rendering. +The `BeautifulSoupCrawler` is a plain HTTP crawler that parses HTML using the well-known [BeautifulSoup](https://pypi.org/project/beautifulsoup4/) library. It fetches pages with an HTTP client that mimics a browser. It's fast and light, but it can't render JavaScript. ### ParselCrawler -The `ParselCrawler` is similar to the `BeautifulSoupCrawler` but uses the [Parsel](https://pypi.org/project/parsel/) library for HTML parsing. Parsel is a lightweight library that provides a CSS selector-based API for extracting data from HTML documents. If you are familiar with the [Scrapy](https://scrapy.org/) framework, you will feel right at home with Parsel. As with the `BeautifulSoupCrawler`, the `ParselCrawler` cannot handle JavaScript rendering. +The `ParselCrawler` is similar to the `BeautifulSoupCrawler` but uses the [Parsel](https://pypi.org/project/parsel/) library for HTML parsing. Parsel extracts data from HTML with CSS selectors and XPath, and it's the selector library [Scrapy](https://scrapy.org/) uses. Like the `BeautifulSoupCrawler`, the `ParselCrawler` can't render JavaScript. ### PlaywrightCrawler -The `PlaywrightCrawler` uses a headless browser controlled by the [Playwright](https://playwright.dev/) library. It can manage Chromium, Firefox, Webkit, and other browsers. Playwright is the successor to the [Puppeteer](https://pptr.dev/) library and is becoming the de facto standard in headless browser automation. If you need a headless browser, choose Playwright. +The `PlaywrightCrawler` uses a headless browser controlled by the [Playwright](https://playwright.dev/) library. It can drive Chromium, Firefox, and WebKit. Choose it when the target pages render their content with JavaScript. ## Installation -Crawlee is available the [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. This package includes the core functionality, while additional features are available as optional extras to keep dependencies and package size minimal. +Crawlee is available as the [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. The package includes the core functionality, and additional features are available as optional extras to keep dependencies small. -You can install Crawlee with all features or choose only the ones you need. For installing it using the [pip](https://pip.pypa.io/en/stable/) package manager, run the following command: +To install Crawlee with all extras using [pip](https://pip.pypa.io/en/stable/), run: ```sh python -m pip install 'crawlee[all]' @@ -57,13 +57,13 @@ Verify that Crawlee is successfully installed: python -c 'import crawlee; print(crawlee.__version__)' ``` -If you plan to use the `PlaywrightCrawler`, you'll need to install Playwright dependencies, including the browser binaries. To do this, run the following command: +If you plan to use the `PlaywrightCrawler`, also install the Playwright browser binaries: ```sh playwright install ``` -For detailed installation instructions, see the [Setting up](./introduction/setting-up) documentation page. +For detailed installation instructions, see [Setting up](./introduction/setting-up). ## Crawling @@ -87,33 +87,33 @@ Run the following example to perform a recursive crawl of the Crawlee website us -When you run the example, you will see Crawlee automating the data extraction process in your terminal. +When you run the example, the crawler logs its progress to your terminal. {/*TODO: improve the logging and add here a sample*/} ## Running headful browser -By default, browsers controlled by Playwright run in headless mode (without a visible window). However, you can configure the crawler to run in a headful mode, which is useful during the development phase to observe the browser's actions. You can also switch from the default Chromium browser to Firefox or WebKit. +By default, browsers controlled by Playwright run in headless mode, without a visible window. During development, it helps to run the browser in headful mode and watch what it does. You can also switch from the default Chromium browser to Firefox or WebKit. {PlaywrightCrawlerHeadfulExample} -When you run the example code, you'll see an automated browser navigating through the Crawlee website. +When you run the example, a browser window opens and navigates through the Crawlee website. {/*TODO: add video example*/} ## Results -By default, Crawlee stores data in the `./storage` directory within your current working directory. The results of your crawl will be saved as JSON files under `./storage/datasets/default/`. +By default, Crawlee stores data in the `./storage` directory within your current working directory. The crawl results are saved as JSON files under `./storage/datasets/default/`. -To view the results, you can use the `cat` command: +To view the results, use the `cat` command: ```sh cat ./storage/datasets/default/000000001.json ``` -The JSON file will contain data similar to the following: +The JSON file contains data similar to the following: ```json { @@ -124,12 +124,12 @@ The JSON file will contain data similar to the following: :::tip -If you want to change the storage directory, you can set the `CRAWLEE_STORAGE_DIR` environment variable to your preferred path. +To change the storage directory, set the `CRAWLEE_STORAGE_DIR` environment variable to your preferred path. ::: -## Examples and further reading +## Next steps -For more examples showcasing various features of Crawlee, visit the [Guides](./guides) section of the documentation. To get a deeper understanding of Crawlee and its components, read the step-by-step [Introduction](./introduction) guide. - -[//]: # (TODO: add related links once they are ready) +- To build a complete scraper for a real website, follow the [Tutorial](./introduction/tutorial). +- To learn how crawlers, requests, storages, and other parts of Crawlee work, read the [Concepts](./concepts). +- For solutions to specific tasks, see the [Guides](./guides). From 92681d20f6c65707b1451713e34f81c54fbb9f09 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 12:31:04 +0200 Subject: [PATCH 04/11] docs: drop guide-style conclusions from the moved concept pages --- docs/02_concepts/08_pydantic_ai_crawler.mdx | 4 ---- docs/02_concepts/09_stagehand_crawler.mdx | 4 ---- docs/02_concepts/26_extending_crawlee.mdx | 8 +++----- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/docs/02_concepts/08_pydantic_ai_crawler.mdx b/docs/02_concepts/08_pydantic_ai_crawler.mdx index 50da4115c9..b2c612b802 100644 --- a/docs/02_concepts/08_pydantic_ai_crawler.mdx +++ b/docs/02_concepts/08_pydantic_ai_crawler.mdx @@ -174,7 +174,3 @@ The errors point at the cause: `UnexpectedModelBehavior`, `ModelHTTPError`, and `UsageLimitExceeded` come from the model run and share the base `AgentRunError`, so catch it to handle any model failure in one place. `ValueError` is raised by the extractor itself, around the schema or `scope`, not by the model. An extraction error is an ordinary request error, so Crawlee retries the request up to `max_request_retries` (3 by default), re-running the extraction and repeating its token cost. For errors that won't pass on retry, such as a wrong API key or an unsupported schema, the retries only waste effort, so catch them in the handler to skip the page, or set `abort_on_error=True` to stop the crawl. - -## Conclusion - -This guide introduced `PydanticAiCrawler` and its `extract` helper, the `PydanticAiDirectExtractor` and `PydanticAiSelectorExtractor` strategies, the built-in and custom distillers, the extract options, how failures and cost are handled, and how to debug extraction. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/02_concepts/09_stagehand_crawler.mdx b/docs/02_concepts/09_stagehand_crawler.mdx index 0abbd98180..34ae27da0f 100644 --- a/docs/02_concepts/09_stagehand_crawler.mdx +++ b/docs/02_concepts/09_stagehand_crawler.mdx @@ -121,7 +121,3 @@ Browserbase credentials (`browserbase_api_key`, `project_id`) can also be provid ## Browser configuration limitations Because Stagehand manages the browser session internally via CDP, only Chromium is supported. Browser settings are limited to the subset accepted by Stagehand's `BrowserLaunchOptions` - `headless`, `args`, `viewport`, `proxy`, `locale`, `executable_path`, and a few others. Features like full browser fingerprinting (canvas, WebGL, screen properties) and incognito pages are not supported. Fingerprint-consistent HTTP headers (`User-Agent`, `Accept`, `sec-ch-ua`) are still injected automatically. - -## Conclusion - -This guide introduced `StagehandCrawler` and its AI page operations: `act()`, `extract()`, `observe()`, and `execute()`. You learned how to configure Stagehand via `StagehandOptions` and switch to Browserbase for cloud browser execution. If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/02_concepts/26_extending_crawlee.mdx b/docs/02_concepts/26_extending_crawlee.mdx index e7e755f1d5..f9f1741aa2 100644 --- a/docs/02_concepts/26_extending_crawlee.mdx +++ b/docs/02_concepts/26_extending_crawlee.mdx @@ -8,7 +8,7 @@ import ApiLink from '@site/src/components/ApiLink'; Crawlee covers the common cases out of the box, but sooner or later you'll hit something it doesn't do: a parser for a format no built-in crawler understands, an HTTP backend your company mandates, a database you want your data in, or a browser that isn't launched through the standard Playwright path. Rather than forking, you can plug your own implementation into the layer that differs and keep everything else. -This guide covers the extension points, the contract each one defines, and links to the guides that go deeper on each. If you maintain a third-party integration, you can build it against these contracts and host your own guide for it. +This page covers the extension points, the contract each one defines, and links to the pages that go deeper on each. If you maintain a third-party integration, you can build it against these contracts and host your own documentation for it. ## Extension points @@ -124,7 +124,7 @@ The context exposes the parsed data to handlers, and the crawler ties the parser Browser crawlers use the same orchestration with a browser-backed context. Extend `PlaywrightCrawler` when an integration needs crawler-level browser behavior or a different handler context. `StagehandCrawler` is an example. It extends `PlaywrightCrawler` with a Stagehand-specific context and browser behavior. If only browser launch or lifecycle differs, a browser plugin is the narrower extension point. -See the [HTTP crawlers](./http-crawlers) page for a worked example built on `selectolax`, and the [Stagehand crawler guide](./stagehand-crawler) for `StagehandCrawler` itself. The [Architecture overview](./architecture-overview) explains how HTTP and browser crawlers relate to the other components. +See the [HTTP crawlers](./http-crawlers) page for a worked example built on `selectolax`, and the [Stagehand crawler](./stagehand-crawler) page for `StagehandCrawler` itself. The [Architecture overview](./architecture-overview) explains how HTTP and browser crawlers relate to the other components. ### HTTP clients @@ -184,8 +184,6 @@ Start with configuration before writing a subclass. You can parse a response wit When more than one fits, pick the narrowest. A custom HTTP client works with every HTTP crawler, so it's easier to maintain than a crawler subclass that hard-codes the same transport. -## Conclusion +## Contract stability Each extension point has a documented class contract, and everything built on top of it keeps working once you implement that contract. Crawlee changes those contracts only in a major release, which makes them the stable surface for a third-party integration and its documentation. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! From 1ef046a66348821c8ca2e4ece8ff746711f598a9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 12:31:18 +0200 Subject: [PATCH 05/11] docs: add concept pages for requests, crawling context, events, statistics, and configuration --- docs/02_concepts/02_requests.mdx | 89 +++++++++++++++++++ docs/02_concepts/03_crawling_context.mdx | 74 +++++++++++++++ docs/02_concepts/21_events.mdx | 75 ++++++++++++++++ docs/02_concepts/22_statistics.mdx | 79 ++++++++++++++++ docs/02_concepts/24_configuration.mdx | 85 ++++++++++++++++++ .../configuration/crawler_configuration.py | 33 +++++++ .../configuration/global_configuration.py | 37 ++++++++ .../crawling_context/buffered_results.py | 29 ++++++ .../crawling_context/context_helpers.py | 48 ++++++++++ .../code_examples/events/listen_to_events.py | 45 ++++++++++ .../code_examples/events/use_state.py | 26 ++++++ .../requests/label_and_user_data.py | 48 ++++++++++ .../requests/unique_key_deduplication.py | 47 ++++++++++ .../statistics/custom_statistics.py | 39 ++++++++ .../statistics/read_final_statistics.py | 34 +++++++ 15 files changed, 788 insertions(+) create mode 100644 docs/02_concepts/02_requests.mdx create mode 100644 docs/02_concepts/03_crawling_context.mdx create mode 100644 docs/02_concepts/21_events.mdx create mode 100644 docs/02_concepts/22_statistics.mdx create mode 100644 docs/02_concepts/24_configuration.mdx create mode 100644 docs/02_concepts/code_examples/configuration/crawler_configuration.py create mode 100644 docs/02_concepts/code_examples/configuration/global_configuration.py create mode 100644 docs/02_concepts/code_examples/crawling_context/buffered_results.py create mode 100644 docs/02_concepts/code_examples/crawling_context/context_helpers.py create mode 100644 docs/02_concepts/code_examples/events/listen_to_events.py create mode 100644 docs/02_concepts/code_examples/events/use_state.py create mode 100644 docs/02_concepts/code_examples/requests/label_and_user_data.py create mode 100644 docs/02_concepts/code_examples/requests/unique_key_deduplication.py create mode 100644 docs/02_concepts/code_examples/statistics/custom_statistics.py create mode 100644 docs/02_concepts/code_examples/statistics/read_final_statistics.py diff --git a/docs/02_concepts/02_requests.mdx b/docs/02_concepts/02_requests.mdx new file mode 100644 index 0000000000..c8ae914227 --- /dev/null +++ b/docs/02_concepts/02_requests.mdx @@ -0,0 +1,89 @@ +--- +id: requests +title: Requests +description: How Crawlee represents a single request, how it carries data between handlers, how it's deduplicated, retried, and tracked through its lifecycle. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import LabelAndUserDataExample from '!!raw-loader!roa-loader!./code_examples/requests/label_and_user_data.py'; +import UniqueKeyDeduplicationExample from '!!raw-loader!roa-loader!./code_examples/requests/unique_key_deduplication.py'; + +A `Request` describes one unit of work for a crawler: the URL to fetch, how to fetch it, and any data your code wants to keep with it. Crawlers read requests from a request manager such as a `RequestQueue`, pass each one to a request handler, and record the result back in the queue. The request object travels with the work the whole time, including its retries, so it's the natural place to store anything a later step needs. + +## Creating requests + +The recommended constructor is `Request.from_url`. It validates the URL (only HTTP and HTTPS are accepted) and computes the unique key used for deduplication. Most methods that accept requests also accept plain URL strings. That includes `crawler.run`, `crawler.add_requests`, and `context.add_requests`. Crawlee turns each string into a request with `Request.from_url(url)` and default options. + +The main fields of a request are: + +- `url`: the URL to crawl. After the request is processed, `loaded_url` holds the final URL if the server redirected. +- `method`: the HTTP method, `GET` by default. +- `headers`: HTTP headers sent with the request, as `HttpHeaders`. You can pass a plain `dict` to `from_url`. +- `payload`: the request body as `bytes`. A `str` passed to `from_url` is encoded to bytes. +- `user_data`: a mutable mapping with your own JSON-serializable data. +- `label`: a string stored in `user_data` that the router uses to pick a handler. + +The `url`, `method`, `payload`, and `unique_key` fields are frozen. To change them, create a new request. + +## Labels and user data + +The `label` decides which handler processes a request. The `Router` looks up a handler registered for the request's label and falls back to the default handler when there's no label or no matching handler. For details, see [Request router](./request-router). + +The `user_data` mapping carries data from the handler that created a request to the handler that processes it. It's stored with the request in the queue, so it survives retries and crawler restarts with a persistent storage. Values must be JSON-serializable. Crawlee also keeps its own metadata under the reserved `__crawlee` key of `user_data`, so don't write to that key. + +Both `label` and `user_data` can be set on a single request with `from_url`, or on every link enqueued from a page with `context.enqueue_links` and `context.add_requests`. In the following example, the start request carries a `source` value, and the default handler enqueues the links it finds with the `DETAIL` label and extra user data that the detail handler reads. + + + {LabelAndUserDataExample} + + +## Unique key and deduplication + +Every request has a `unique_key`. A request queue stores at most one request per unique key, so adding a request with a key that's already in the queue does nothing. The `ProcessedRequest` returned by `add_request` tells you whether that happened through its `was_already_present` field. Deduplication covers handled requests too, so a page that was already crawled isn't crawled again when another page links to it. + +By default, `from_url` computes the unique key by normalizing the URL. The normalization lowercases the URL, removes `utm_*` query parameters, sorts the remaining query parameters, removes a trailing slash from the path, and drops the URL fragment. The following options of `from_url` change the computation: + +- `keep_url_fragment=True` keeps the fragment, so `/page#a` and `/page#b` become two requests. It's useful for single-page apps that route with fragments. +- `use_extended_unique_key=True` adds the HTTP method, a hash of the payload, a hash of selected headers (`Accept`, `Accept-Language`, `Authorization`, and `Content-Type`), and the session ID to the key. Use it when requests to the same URL differ only by their body, as with POST APIs. +- `always_enqueue=True` prepends a random ID to the key, so the request is added even if the same URL is already in the queue. You can't combine it with a custom `unique_key`. +- `unique_key='...'` sets the key directly and skips the computation. + +The next example adds several requests to a queue and prints whether each one was a duplicate, followed by its unique key. + + + {UniqueKeyDeduplicationExample} + + +The second request is a duplicate of the first one, so the queue ends up with five requests. + +## Retries + +When a request handler raises an exception, the crawler retries the request. It increments `retry_count` and returns the request to the queue. A request is retried until `retry_count` reaches the crawler's `max_request_retries` option, which defaults to 3. To set a different limit for a single request, pass `max_retries` to `from_url`. A request created with `no_retry=True` isn't retried at all, and neither are requests that failed with an HTTP client error status (4xx). Session rotations caused by blocking are counted separately and are limited by `max_session_rotations`. + +When the retries run out, the crawler calls the failed request handler and marks the request as handled. For error handlers, failed request handlers, and the related options, see [Error handling](./error-handling). + +## Crawl depth + +The `crawl_depth` property tells how many links away from a start request a request is. Requests you pass to `run` or `add_requests` on the crawler have depth 0. Each request created by `context.enqueue_links` or `context.add_requests` gets the depth of the current request plus one. + +The crawler's `max_crawl_depth` option limits how deep the crawl goes. Requests at the maximum depth are still processed, but links found on their pages aren't enqueued. By default, the depth isn't limited. + +## Binding to a session + +Pass `session_id` to `from_url` to bind a request to one session from the crawler's `SessionPool`. The crawler then processes the request with that session only. If the session isn't available anymore, the request fails with a `RequestCollisionError` and isn't retried. For details on sessions, see [Session management](./session-management). + +## Request lifecycle + +Crawlee tracks the progress of a request in its `state` property, a `RequestState` value. A request that's processed without errors moves through the following states: + +1. `UNPROCESSED`: the request was created and waits in the queue. +2. `BEFORE_NAV`: the crawler took the request and runs pre-navigation hooks. +3. `AFTER_NAV`: the HTTP response or browser page is ready. Crawlers that don't navigate, such as `BasicCrawler`, skip this state. +4. `REQUEST_HANDLER`: the router runs your request handler. +5. `DONE`: the handler finished, and the request is marked as handled. + +When an error occurs, the request switches to `ERROR_HANDLER` while the crawler runs the error handler and decides whether to retry. A retried request goes back to the queue and starts again from `BEFORE_NAV`. A request that runs out of retries ends in the `ERROR` state. Requests that the crawler skips, which currently happens only for URLs disallowed by robots.txt, end in the `SKIPPED` state. For details, see [Respecting robots.txt](../guides/respect-robots-txt-file). + +Failed and skipped requests are marked as handled too, so the queue doesn't hand them out again. The `handled_at` field holds the time a request was marked as handled, and `was_already_handled` tells whether it's set. diff --git a/docs/02_concepts/03_crawling_context.mdx b/docs/02_concepts/03_crawling_context.mdx new file mode 100644 index 0000000000..b58f6dd3d0 --- /dev/null +++ b/docs/02_concepts/03_crawling_context.mdx @@ -0,0 +1,74 @@ +--- +id: crawling-context +title: Crawling context +description: What a request handler receives, how each crawler extends the basic crawling context, and how the context helpers store results. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ContextHelpersExample from '!!raw-loader!roa-loader!./code_examples/crawling_context/context_helpers.py'; +import BufferedResultsExample from '!!raw-loader!roa-loader!./code_examples/crawling_context/buffered_results.py'; + +Every request handler receives a single argument, the crawling context. It holds the request being processed, the session and proxy used for it, and a set of helper functions for storing data and adding new requests. The crawler creates a new context for each request, so handlers don't share it. + +## Context types + +All contexts extend `BasicCrawlingContext`, which is what `BasicCrawler` passes to its handlers. Each crawler builds on it and adds the attributes that make sense for the way it fetches pages: + +- `HttpCrawlingContext` adds `http_response`, the response returned by the HTTP client. +- `ParsedHttpCrawlingContext` adds `parsed_content` together with the `enqueue_links` and `extract_links` helpers. `HttpCrawler` uses it with the raw response body as `parsed_content`. +- `BeautifulSoupCrawlingContext` and `ParselCrawlingContext` are parsed contexts that expose the parsed document as `soup` and `selector`. +- `PlaywrightPreNavCrawlingContext` is what Playwright pre-navigation hooks receive. It adds the Playwright `page` before navigation, `goto_options` and the `block_requests` helper. +- `PlaywrightCrawlingContext` is what `PlaywrightCrawler` handlers receive. It adds the navigation `response`, `enqueue_links`, `extract_links` and `infinite_scroll`. + +Pre-navigation hooks of HTTP crawlers receive a plain `BasicCrawlingContext`, because no response exists yet. Their post-navigation hooks receive an `HttpCrawlingContext`. Annotate your handler with the context type of your crawler to get the right attributes and type checking. + +For details on the crawler-specific attributes, see [HTTP crawlers](./http-crawlers) and [Playwright crawler](./playwright-crawler). + +## Request, session and proxy + +The `request` attribute is the `Request` being processed. It carries the URL, the label used for routing, user data and the retry count. For details, see [Requests](./requests). + +The `session` attribute is the `Session` assigned to the request, or `None` when the crawler runs without a session pool. You can call `mark_bad()` or `retire()` on it from the handler when a page shows signs of blocking. For details, see [Session management](./session-management). + +The `proxy_info` attribute is a `ProxyInfo` describing the proxy used for the request, or `None` when no proxy is configured. For details, see [Proxy management](./proxy-management). + +The `log` attribute is the crawler's logger, so messages from handlers follow the same log configuration as the crawler's own. For details, see [Logging](./logging). + +## Context helpers + +The context provides helpers that operate on the crawler's storages and request queue: + +- `push_data` stores one item or a list of items in a dataset. It writes to the default dataset unless you pass `dataset_id`, `dataset_name` or `dataset_alias`. +- `add_requests` adds URLs or `Request` objects to the request queue. Relative URLs resolve against the URL of the current page, and each new request gets a crawl depth one higher than the current one. +- `enqueue_links` extracts links from the current page and adds them to the request queue through `add_requests`. It's available on the parsed HTTP contexts and on `PlaywrightCrawlingContext`. For filtering links and choosing an enqueue strategy, see [Crawling links](../guides/crawling-links). +- `get_key_value_store` opens a key-value store, the default one unless you pass `id`, `name` or `alias`. +- `use_state` returns a dictionary shared by all handler calls of the crawler. The crawler keeps it in the default key-value store and saves it automatically. This helper is experimental and its interface may change. +- `send_request` sends an extra HTTP request through the crawler's HTTP client, with the session and proxy of the current request. Its response isn't parsed, and the request doesn't go through the request queue. + +The following example uses most of these helpers in a `ParselCrawler` handler and reads the results back from the crawler once the run finishes. + + + {ContextHelpersExample} + + +## Results are committed after the handler succeeds + +The `push_data`, `add_requests` and `enqueue_links` calls and the `set_value` calls on a key-value store from `get_key_value_store` don't write anything right away. The context records them in an internal `RequestHandlerRunResult` object, and the crawler commits the recorded changes only after the request handler returns without an exception. Reading a key from that key-value store in the same handler returns the value you've set, even before the commit. + +When the handler raises, the crawler discards the recorded changes and retries the request. A handler that pushes an item and then fails on the next line doesn't leave a partial or duplicate item in the dataset. The retry starts with an empty record and produces the output once, when it succeeds. The same applies to requests added by a failed attempt. + +The example simulates a failure on the first attempt. The dataset ends up with a single item, the one from the second attempt. + + + {BufferedResultsExample} + + +A few parts of the context write directly, outside of the buffer: + +- Changes to the dictionary from `use_state` are visible to other handlers immediately and stay in place when the handler fails. +- Requests made with `send_request` happen when you call it. +- Storages you open yourself, for example with `Dataset.open`, write immediately. + +Error handlers and failed request handlers receive a copy of the context whose `push_data`, `add_requests` and `get_key_value_store` write directly, because the request has already failed and there's no successful run to wait for. For details, see [Error handling](./error-handling). diff --git a/docs/02_concepts/21_events.mdx b/docs/02_concepts/21_events.mdx new file mode 100644 index 0000000000..93bd6562ab --- /dev/null +++ b/docs/02_concepts/21_events.mdx @@ -0,0 +1,75 @@ +--- +id: events +title: Events and state persistence +description: How Crawlee's event manager broadcasts system and lifecycle events, how to listen to them, and how components persist their state on the persist state event. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ListenToEventsExample from '!!raw-loader!roa-loader!./code_examples/events/listen_to_events.py'; +import UseStateExample from '!!raw-loader!roa-loader!./code_examples/events/use_state.py'; + +Crawlee components talk to each other through events. An event manager emits events such as "save your state now" or "here's the current CPU and memory usage", and any component can register a listener for them. The crawler doesn't need to know which components care about an event. The `Snapshotter` listens to system info events to drive autoscaling, and statistics, storages, and your own code listen to persist state events to save their progress. + +## Event manager + +The `EventManager` class keeps a registry of listeners and invokes them whenever an event is emitted. It also emits the `PERSIST_STATE` event on its own at a regular interval. + +Locally, Crawlee uses the `LocalEventManager` subclass, which additionally measures CPU and memory usage and emits it as the `SYSTEM_INFO` event. The intervals come from the `Configuration`: + +- `persist_state_interval` defaults to 1 minute. You can set it with the `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` environment variable. +- `system_info_interval` defaults to 1 second. You can set it with the `CRAWLEE_SYSTEM_INFO_INTERVAL_MILLIS` environment variable. + +The event manager is one of the global services of the [service locator](./service-locator). Call `service_locator.get_event_manager()` to get it. If none was set, the service locator creates a `LocalEventManager` from the current configuration on first access. To use a different one, register it with `service_locator.set_event_manager()` before you create the crawler. A crawler also accepts an `event_manager` argument, but components such as statistics and storages subscribe through the global service locator, so the global registration is the one to change. + +The event manager is an async context manager. The crawler enters it when `run()` starts and exits it when the run finishes, so events are only emitted while a crawler runs. Outside a crawler, use `async with event_manager:` yourself. + +## Events + +The `Event` enum lists the event names. Each event comes with a payload model: + +- `PERSIST_STATE` carries `EventPersistStateData`. It's emitted when the event manager starts, then at every `persist_state_interval`, and one last time when it closes. Its `is_migrating` flag tells whether the process is about to be migrated. +- `SYSTEM_INFO` carries `EventSystemInfoData` with the CPU usage ratio and the memory used by the current process and its children. The `LocalEventManager` emits it at every `system_info_interval`. +- `CRAWLER_STATUS` carries `EventCrawlerStatusData` with the same status message that the crawler periodically logs. +- `MIGRATING`, `ABORTING`, and `EXIT` carry `EventMigratingData`, `EventAbortingData`, and `EventExitData`. Crawlee itself never emits them locally. They come from a platform, for example from the event manager of the [Apify SDK](https://docs.apify.com/sdk/python) when an Actor run is migrated to another server or aborted. + +The enum also defines session pool and browser pool event names, such as `SESSION_RETIRED` or `BROWSER_LAUNCHED`, which Crawlee for Python doesn't emit. + +## Listening to events + +Register a listener with `on()` and remove it with `off()`. Calling `off()` without a listener removes all listeners of the event. A listener can be sync or async and accepts either the event payload or no argument at all. Each invocation runs in its own task, and sync listeners run in a separate thread, so a slow listener doesn't block the crawler. An exception raised by a listener is logged and doesn't affect other listeners. + +The following example registers an event manager with a shorter persist state interval and prints the payload of the persist state and system info events during a short crawl: + + + {ListenToEventsExample} + + +## State persistence + +State persistence builds on the `PERSIST_STATE` event. Components that need to survive a restart or a migration keep their state in memory, save it to a `KeyValueStore` whenever the event fires, and load it back on the next start. The default key-value store of a crawler run then holds records like these: + +- `__CRAWLER_STATISTICS_` holds the crawler's [statistics](./statistics). The crawler's default `Statistics` instance has persistence enabled. +- `__RQ_STATE_` holds the ordering and in-progress information of a request queue backed by the file system storage client. The requests themselves are stored as individual files. +- `CRAWLEE_STATE_` holds the state you manage with `use_state`, as described in [Crawler state](#crawler-state). + +Other components persist their state when you ask them to. The `SessionPool` does so with `persistence_enabled=True`, and the `RequestList` with a `persist_state_key`. For details, see [Request loaders](./request-loaders). + +Internally, these components use a helper class that loads the state on start, subscribes to `PERSIST_STATE`, and saves the state one last time on teardown. The helper isn't part of the public API. In your own code, use `use_state`, or listen to `PERSIST_STATE` and save your data to a key-value store. + +Persisted state only helps if it's still there on the next run. Crawlee purges the default storages on start, so to resume an interrupted crawl, disable `purge_on_start`. For details, see [Stopping and resuming crawlers](../guides/stopping-and-resuming-crawlers). + +## Crawler state + +The `use_state` helper of the crawling context returns a dictionary that's saved automatically. Every call within a crawler returns the same dictionary, so you can read and update it from any request handler. Crawlee saves it to the default key-value store on every `PERSIST_STATE` event and when the run finishes, and loads it back from there when the crawler starts again with existing storage. The optional argument sets the initial value when no saved state exists yet. + +The dictionary is stored under the `CRAWLEE_STATE_` key, where the id is a counter of crawler instances in the process. Pass the same `id` to the constructors of several crawlers to make them share their state. Outside request handlers, use `crawler.use_state()`. Values have to be JSON serializable. + +The following example stores the page title of every visited URL in the crawler state and reads it after the run: + + + {UseStateExample} + + +`use_state` is an experimental feature, so its behavior and interface may change in future versions. diff --git a/docs/02_concepts/22_statistics.mdx b/docs/02_concepts/22_statistics.mdx new file mode 100644 index 0000000000..434ca613ac --- /dev/null +++ b/docs/02_concepts/22_statistics.mdx @@ -0,0 +1,79 @@ +--- +id: statistics +title: Statistics +description: How Crawlee tracks request counts, retries, durations, and errors during a crawl, and how to read, log, and persist them. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import ReadFinalStatisticsExample from '!!raw-loader!roa-loader!./code_examples/statistics/read_final_statistics.py'; +import CustomStatisticsExample from '!!raw-loader!roa-loader!./code_examples/statistics/custom_statistics.py'; + +Every crawler collects runtime statistics about the requests it processes. It counts finished and failed requests, measures how long they take, records how many retries each one needed, and groups the errors it hits. The `Statistics` class does the bookkeeping. The crawler calls it whenever a request starts, finishes, fails, or gets retried, so you don't have to record anything yourself. + +## What gets tracked + +The raw data lives in a `StatisticsState` model. Its main fields are: + +- `requests_finished` and `requests_failed` count requests that succeeded or failed after exhausting their retries. +- `requests_retries` counts requests that needed at least one retry, and `request_retry_histogram` shows how many requests finished after 0, 1, 2, and more retries. +- `request_total_finished_duration`, `request_total_failed_duration`, `request_min_duration`, and `request_max_duration` measure request processing time. +- `requests_with_status_code` maps HTTP status codes to the number of responses that returned them. +- `crawler_started_at`, `crawler_finished_at`, and the computed `crawler_runtime` describe the run itself. + +From the state, `Statistics.calculate` derives a `FinalStatistics` object. It's a frozen summary with the request counts, the retry histogram as a list, average finished and failed durations, total duration, per-minute rates for finished and failed requests, and the crawler runtime. The same summary appears in the periodic and final statistics logs. + +## Reading statistics + +There are two ways to get the statistics of a crawl: + +- `crawler.run()` returns the `FinalStatistics` of the run once it ends. +- `crawler.statistics` gives you the `Statistics` instance, so you can read the full `state` or call `calculate()` at any time, including from a request handler while the crawl is running. + +The following example reads the summary returned by `run()`, then looks at state fields that `FinalStatistics` doesn't include, and the errors collected during the run. + + + {ReadFinalStatisticsExample} + + +A crawler that you run again with `run()` resets its statistics at the start of the new run. To keep the numbers from a previous run, store the returned `FinalStatistics` before calling `run()` again. + +## Periodic logging + +While the crawler runs, `Statistics` logs the current `FinalStatistics` at a fixed interval, and the crawler logs a final summary when the run ends. Two options control the periodic messages, and both belong to `Statistics`: + +- `log_interval` sets how often the statistics are logged. The default is one minute. +- `statistics_log_format` picks between `table`, a formatted table, and `inline`, a single message with the values attached as extra log fields. + +The crawler's constructor also accepts `statistics_log_format`. When the crawler creates its own `Statistics`, it passes the value through, so the option controls both the periodic and the final logs. When you pass a custom `Statistics`, the crawler's option applies only to the final summary. For more about log output, including structured JSON logs, see [Logging](./logging). + +## Persistence + +`Statistics` can save its state to a key-value store, so a crawl that gets migrated, aborted, or restarted continues counting from where it left off. When persistence is on, the state is loaded from the store when the crawl starts, saved on every `PERSIST_STATE` event, and saved once more when the crawl ends. For details about the event, see [Events](./events). + +The following options control persistence: + +- `persistence_enabled` turns persistence on. With `'explicit_only'`, a saved state is loaded at the start, but nothing is saved automatically, neither on `PERSIST_STATE` events nor at the end of the crawl. +- `persist_state_key` is the key the state is stored under. It defaults to `__CRAWLER_STATISTICS_`, where `` is a per-process counter of `Statistics` instances. +- `persist_state_kvs_name` selects a named key-value store. Without it, the default key-value store is used. + +The statistics a crawler creates for itself have persistence enabled and use the crawler's key-value store. A `Statistics` instance you create yourself has `persistence_enabled=False` by default. For how to resume an interrupted crawl, see [Stopping and resuming crawlers](../guides/stopping-and-resuming-crawlers). + +## Custom statistics + +To change any of these options, create a `Statistics` instance and pass it to the crawler through the `statistics` argument. Use `Statistics.with_default_state` to get an instance with the standard `StatisticsState` model. + +The following example logs the statistics every second in the `inline` format, persists them under a custom key, and reads the persisted record back after the run. + + + {CustomStatisticsExample} + + +A custom `Statistics` logs its periodic messages through the `crawlee.statistics._statistics` logger. To log through the crawler's logger instead, pass it as `periodic_message_logger`. + +## Error tracking + +Each `Statistics` instance holds an `error_tracker`, which groups errors from failed request attempts by their type, the place in the code where they were raised, and a similar message. Its `total` and `unique_error_count` properties count all errors and the distinct groups, and `get_most_common_errors()` returns the most frequent groups with their counts. At the end of a run that had errors, the crawler logs both counts. A second tracker, `error_tracker_retry`, collects session errors that caused a retry with a new session. + +With `save_error_snapshots=True`, the error tracker also saves a snapshot of the page on the first occurrence of each unique error. For details and examples, see [Capturing page snapshots on errors](./error-handling#capturing-page-snapshots-on-errors). diff --git a/docs/02_concepts/24_configuration.mdx b/docs/02_concepts/24_configuration.mdx new file mode 100644 index 0000000000..2f89904de4 --- /dev/null +++ b/docs/02_concepts/24_configuration.mdx @@ -0,0 +1,85 @@ +--- +id: configuration +title: Configuration +description: How Crawlee's Configuration class resolves settings from code and environment variables, its most useful options, and how to apply it globally or per crawler. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; + +import GlobalConfigurationExample from '!!raw-loader!roa-loader!./code_examples/configuration/global_configuration.py'; +import CrawlerConfigurationExample from '!!raw-loader!roa-loader!./code_examples/configuration/crawler_configuration.py'; + +The `Configuration` class holds the settings that Crawlee components share: where storages live, the log level, resource limits for autoscaling, browser launch defaults, and event intervals. Every option has a default, so a crawler runs without any configuration. You change a setting when the defaults don't fit your environment, for example to keep storages between runs or to cap memory in a container. + +## How values resolve + +`Configuration` is a [pydantic-settings](https://docs.pydantic.dev/latest/concepts/pydantic_settings/) model. Each value is resolved when the instance is created, in this order: + +1. A keyword argument passed to the constructor, such as `Configuration(storage_dir='./data')`. +2. An environment variable. +3. The default value. + +Each option declares its environment variables explicitly. Most options accept two names, `CRAWLEE_` and `APIFY_`, so the same code reads the variables the Apify platform sets for an Actor. Names are case-insensitive. If both variables are set, the `APIFY_` one wins. Some variable names differ from the field name. For example `storage_dir` maps to `CRAWLEE_STORAGE_DIR` and `APIFY_LOCAL_STORAGE_DIR`, and interval options use a `_MILLIS` suffix and take milliseconds. + +The constructor accepts the field name or any of its aliases. Interval options take a `timedelta`. Values coming from the environment are converted and validated, so `CRAWLEE_HEADLESS=false` becomes `False` and an out-of-range ratio raises a validation error. + +Because values are read at creation time, set environment variables before Crawlee creates its configuration. That happens on first use, when a crawler or a storage asks the [service locator](./service-locator) for it. + +## Most useful options + +The following table lists the options you'll reach for most often. The `Configuration` API reference documents all of them. + +| Option | Environment variable | Default | Used for | +| --- | --- | --- | --- | +| `storage_dir` | `CRAWLEE_STORAGE_DIR` | `'./storage'` | Root directory of the file system and SQLite storage clients. | +| `purge_on_start` | `CRAWLEE_PURGE_ON_START` | `True` | Purging the default and unnamed storages when they're opened. | +| `log_level` | `CRAWLEE_LOG_LEVEL` | `'INFO'` | Level of the logger that the crawler configures. | +| `memory_mbytes` | `CRAWLEE_MEMORY_MBYTES` | `None` | Absolute memory budget for autoscaling. | +| `available_memory_ratio` | `CRAWLEE_AVAILABLE_MEMORY_RATIO` | `0.25` | Share of the available memory used as the budget when `memory_mbytes` isn't set. | +| `max_used_memory_ratio` | `CRAWLEE_MAX_USED_MEMORY_RATIO` | `0.9` | Share of the memory budget above which the system counts as overloaded. | +| `max_used_cpu_ratio` | `CRAWLEE_MAX_USED_CPU_RATIO` | `0.95` | CPU usage above which the system counts as overloaded. | +| `headless` | `CRAWLEE_HEADLESS` | `True` | Launching Playwright browsers in headless mode. | +| `default_browser_path` | `CRAWLEE_DEFAULT_BROWSER_PATH` | `None` | Path to a browser executable for Playwright. | +| `disable_browser_sandbox` | `CRAWLEE_DISABLE_BROWSER_SANDBOX` | `False` | Disabling the Chromium sandbox, which some containers require. | +| `persist_state_interval` | `CRAWLEE_PERSIST_STATE_INTERVAL_MILLIS` | 1 minute | Interval of `PersistState` events that trigger state persistence. | +| `system_info_interval` | `CRAWLEE_SYSTEM_INFO_INTERVAL_MILLIS` | 1 second | Interval of `SystemInfo` events with CPU and memory usage. | +| `internal_timeout` | `CRAWLEE_INTERNAL_TIMEOUT` | `None` | Timeout of internal operations. When unset, the crawler uses twice the request handler timeout, at least 5 minutes. | + +Every variable in the table also has an `APIFY_` counterpart, except `CRAWLEE_INTERNAL_TIMEOUT`. `memory_mbytes` additionally reads `ACTOR_MEMORY_MBYTES`, which takes precedence over the other two. + +Some options behave differently depending on whether you set them: + +- `log_level` takes effect only when you set it explicitly. Otherwise the crawler logs at `INFO`, or at `DEBUG` when Python runs in [development mode](https://docs.python.org/3/library/devmode.html). For details, see [Logging](./logging). +- `purge_on_start` doesn't touch named storages, which are meant to outlive a run. For details, see [Storages](./storages#cleaning-up-the-storages). +- Under a container or other resource limit, the default `available_memory_ratio` makes Crawlee log the resulting memory budget as a warning. Setting `memory_mbytes` or `available_memory_ratio` silences it. For details, see [Scaling crawlers](./scaling-crawlers#running-under-a-resource-limit). + +## Applying a configuration globally + +The `service_locator` singleton holds the global configuration that all Crawlee components use by default. Register your instance with `service_locator.set_configuration()` before you create any crawler or open any storage. `Configuration.get_global_configuration()` returns the same instance as `service_locator.get_configuration()`. + +The following example sets the log level to `WARNING`, so the crawler's `INFO` messages don't show up in the output. + + + {GlobalConfigurationExample} + + +If nothing is registered, the first call to `service_locator.get_configuration()` creates a default `Configuration` from the environment and keeps it. Once the service locator holds a configuration, setting a different instance raises a `ServiceConflictError`. Setting the same instance again is allowed. A crawler created without arguments retrieves the global configuration, so call `set_configuration()` before that. For details, see [Service locator](./service-locator#conflict-prevention). + +`get_global_configuration()` is a class method. Called on a subclass of `Configuration`, it raises a `TypeError` when the global instance isn't of that subclass. + +## Applying a configuration to one crawler + +Pass a `Configuration` to the crawler's `configuration` argument to use it for that crawler only. The crawler keeps it in a service locator of its own and leaves the global one untouched. It uses the configuration for the storages it opens, such as its default request queue and dataset, for the `Snapshotter` that drives autoscaling, and for `internal_timeout`. + + + {CrawlerConfigurationExample} + + +A few settings are always read from the global configuration, whatever you pass to the crawler: + +- The log level that the crawler applies when it configures logging. +- The browser launch defaults `headless`, `default_browser_path`, and `disable_browser_sandbox`. +- The event intervals, unless you also pass an `event_manager`. Without one, the crawler uses the global event manager and logs a warning about it. + +To change these settings, register the configuration globally or configure the component directly, for example with the `headless` argument of `PlaywrightCrawler`. diff --git a/docs/02_concepts/code_examples/configuration/crawler_configuration.py b/docs/02_concepts/code_examples/configuration/crawler_configuration.py new file mode 100644 index 0000000000..4dd358405c --- /dev/null +++ b/docs/02_concepts/code_examples/configuration/crawler_configuration.py @@ -0,0 +1,33 @@ +import asyncio + +from crawlee.configuration import Configuration +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + # Keep this crawler's storages in a separate directory and don't purge them on + # start. A second run finds the request already handled and keeps the dataset. + configuration = Configuration( + storage_dir='./my_crawler_storage', + purge_on_start=False, + ) + + # The configuration applies to this crawler instance only. + crawler = ParselCrawler( + configuration=configuration, + max_requests_per_crawl=1, + ) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + await context.push_data({'url': context.request.url}) + + await crawler.run(['https://crawlee.dev']) + + data = await crawler.get_data() + print(f'Dataset holds {len(data.items)} items.') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/configuration/global_configuration.py b/docs/02_concepts/code_examples/configuration/global_configuration.py new file mode 100644 index 0000000000..ac41a9c288 --- /dev/null +++ b/docs/02_concepts/code_examples/configuration/global_configuration.py @@ -0,0 +1,37 @@ +import asyncio +from datetime import timedelta + +from crawlee import service_locator +from crawlee.configuration import Configuration +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + # Register the configuration before any Crawlee component reads it. + configuration = Configuration( + log_level='WARNING', + persist_state_interval=timedelta(seconds=30), + ) + service_locator.set_configuration(configuration) + + # Any later lookup returns the same instance. + global_configuration = Configuration.get_global_configuration() + print(f'Same instance: {global_configuration is configuration}') + + # The crawler picks up the global configuration. + crawler = ParselCrawler(max_requests_per_crawl=1) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + # Logged at INFO level, so the WARNING threshold filters it out. + context.log.info(f'Processing {context.request.url} ...') + await context.push_data({'title': context.selector.css('title::text').get()}) + + await crawler.run(['https://crawlee.dev']) + + data = await crawler.get_data() + print(data.items) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/crawling_context/buffered_results.py b/docs/02_concepts/code_examples/crawling_context/buffered_results.py new file mode 100644 index 0000000000..ab7a072db6 --- /dev/null +++ b/docs/02_concepts/code_examples/crawling_context/buffered_results.py @@ -0,0 +1,29 @@ +import asyncio + +from crawlee.crawlers import BasicCrawler, BasicCrawlingContext + + +async def main() -> None: + crawler = BasicCrawler(max_request_retries=2) + + @crawler.router.default_handler + async def request_handler(context: BasicCrawlingContext) -> None: + attempt = context.request.retry_count + 1 + context.log.info(f'Attempt {attempt} for {context.request.url}') + + # Buffered until the handler finishes without an exception. + await context.push_data({'url': context.request.url, 'attempt': attempt}) + + # Fail on the first attempt, so the crawler retries the request. + if attempt == 1: + raise RuntimeError('Simulated failure') + + await crawler.run(['https://crawlee.dev/']) + + # Only the item from the successful attempt is stored. + data = await crawler.get_data() + print(data.items) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/crawling_context/context_helpers.py b/docs/02_concepts/code_examples/crawling_context/context_helpers.py new file mode 100644 index 0000000000..9a953f8224 --- /dev/null +++ b/docs/02_concepts/code_examples/crawling_context/context_helpers.py @@ -0,0 +1,48 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=5) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + # The request being processed and the crawler's logger. + context.log.info(f'Processing {context.request.url} ...') + + # Store a result in the default dataset. + await context.push_data( + { + 'url': context.request.url, + 'title': context.selector.css('title::text').get(), + } + ) + + # Store a value in the default key-value store. + kvs = await context.get_key_value_store() + await kvs.set_value('last-url', context.request.url) + + # Keep a counter in the crawler state shared by all handler calls. + state = await context.use_state({'pages': 0}) + pages = state['pages'] + if isinstance(pages, int): + state['pages'] = pages + 1 + + # Add links found on the page to the request queue. + await context.enqueue_links() + + await crawler.run(['https://crawlee.dev/']) + + kvs = await crawler.get_key_value_store() + print(f'Last URL: {await kvs.get_value("last-url")}') + + state = await crawler.use_state() + print(f'Pages counted in state: {state["pages"]}') + + data = await crawler.get_data() + print(f'Items in dataset: {len(data.items)}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/events/listen_to_events.py b/docs/02_concepts/code_examples/events/listen_to_events.py new file mode 100644 index 0000000000..5aef802c3d --- /dev/null +++ b/docs/02_concepts/code_examples/events/listen_to_events.py @@ -0,0 +1,45 @@ +import asyncio +from datetime import timedelta + +from crawlee import service_locator +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext +from crawlee.events import ( + Event, + EventPersistStateData, + EventSystemInfoData, + LocalEventManager, +) + + +async def main() -> None: + # Register an event manager with a shorter persist state interval, so that + # the event fires several times during this short crawl. + event_manager = LocalEventManager(persist_state_interval=timedelta(seconds=2)) + service_locator.set_event_manager(event_manager) + + async def on_persist_state(event_data: EventPersistStateData) -> None: + print(f'Persist state event (is_migrating={event_data.is_migrating})') + + async def on_system_info(event_data: EventSystemInfoData) -> None: + cpu = event_data.cpu_info.used_ratio + memory = event_data.memory_info.current_size + print(f'System info event: CPU {cpu:.0%}, memory {memory}') + + event_manager.on(event=Event.PERSIST_STATE, listener=on_persist_state) + event_manager.on(event=Event.SYSTEM_INFO, listener=on_system_info) + + crawler = ParselCrawler(max_requests_per_crawl=20) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + await context.enqueue_links() + + await crawler.run(['https://crawlee.dev']) + + # Stop listening to the system info events. + event_manager.off(event=Event.SYSTEM_INFO, listener=on_system_info) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/events/use_state.py b/docs/02_concepts/code_examples/events/use_state.py new file mode 100644 index 0000000000..7e68fb6f11 --- /dev/null +++ b/docs/02_concepts/code_examples/events/use_state.py @@ -0,0 +1,26 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=10) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + # Every call returns the same dictionary. Crawlee saves it to the default + # key-value store on each persist state event and at the end of the run. + state = await context.use_state() + state[context.request.url] = context.selector.css('title::text').get() + + await context.enqueue_links() + + await crawler.run(['https://crawlee.dev']) + + # The crawler exposes the same state outside the request handlers. + state = await crawler.use_state() + print(f'Title of the start page: {state["https://crawlee.dev"]}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/requests/label_and_user_data.py b/docs/02_concepts/code_examples/requests/label_and_user_data.py new file mode 100644 index 0000000000..c59c62c885 --- /dev/null +++ b/docs/02_concepts/code_examples/requests/label_and_user_data.py @@ -0,0 +1,48 @@ +import asyncio + +from crawlee import Request +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler( + max_requests_per_crawl=5, + max_crawl_depth=1, + ) + + @crawler.router.default_handler + async def default_handler(context: ParselCrawlingContext) -> None: + source = context.request.user_data['source'] + context.log.info(f'Start page {context.request.url} (source: {source})') + + # Links enqueued here get the `DETAIL` label and carry the same user data. + await context.enqueue_links( + label='DETAIL', + user_data={'source': source, 'parent_url': context.request.url}, + ) + + @crawler.router.handler('DETAIL') + async def detail_handler(context: ParselCrawlingContext) -> None: + request = context.request + context.log.info( + f'Detail page {request.url} ' + f'(depth: {request.crawl_depth}, parent: {request.user_data["parent_url"]})' + ) + await context.push_data( + { + 'url': request.url, + 'title': context.selector.css('title::text').get(), + 'source': request.user_data['source'], + } + ) + + start_request = Request.from_url( + 'https://crawlee.dev/', + user_data={'source': 'homepage'}, + ) + + await crawler.run([start_request]) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/requests/unique_key_deduplication.py b/docs/02_concepts/code_examples/requests/unique_key_deduplication.py new file mode 100644 index 0000000000..83d6a5a446 --- /dev/null +++ b/docs/02_concepts/code_examples/requests/unique_key_deduplication.py @@ -0,0 +1,47 @@ +import asyncio + +from crawlee import Request +from crawlee.storages import RequestQueue + + +async def main() -> None: + request_queue = await RequestQueue.open(alias='deduplication-demo') + + requests = [ + # The unique key is the normalized URL. + Request.from_url('https://crawlee.dev/docs/'), + # Same unique key: trailing slash, `utm_` parameters and fragment are ignored. + Request.from_url('https://CRAWLEE.dev/docs?utm_source=newsletter#intro'), + # A random prefix makes the unique key different every time. + Request.from_url('https://crawlee.dev/docs', always_enqueue=True), + # A custom unique key replaces the computed one. + Request.from_url('https://crawlee.dev/docs', unique_key='docs-second-visit'), + # POST requests to the same URL that differ only by payload. + Request.from_url( + 'https://crawlee.dev/search', + method='POST', + payload='{"query": "crawler"}', + use_extended_unique_key=True, + ), + Request.from_url( + 'https://crawlee.dev/search', + method='POST', + payload='{"query": "storage"}', + use_extended_unique_key=True, + ), + ] + + for request in requests: + processed = await request_queue.add_request(request) + if processed is None: + continue + print(f'{processed.was_already_present!s:<5} {request.unique_key}') + + total = await request_queue.get_total_count() + print(f'Requests in the queue: {total}') + + await request_queue.drop() + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/statistics/custom_statistics.py b/docs/02_concepts/code_examples/statistics/custom_statistics.py new file mode 100644 index 0000000000..4cebddb322 --- /dev/null +++ b/docs/02_concepts/code_examples/statistics/custom_statistics.py @@ -0,0 +1,39 @@ +import asyncio +from datetime import timedelta + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext +from crawlee.statistics import Statistics + + +async def main() -> None: + statistics = Statistics.with_default_state( + # Log the current statistics every second. + log_interval=timedelta(seconds=1), + log_message='Crawler statistics', + # Log values as extra fields of a single message. + statistics_log_format='inline', + # Persist the statistics to the default key-value store. + persistence_enabled=True, + persist_state_key='MY_CRAWLER_STATISTICS', + ) + + crawler = ParselCrawler( + statistics=statistics, + max_requests_per_crawl=50, + ) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + await context.enqueue_links() + + await crawler.run(['https://crawlee.dev']) + + # Read the persisted statistics back from the key-value store. + kvs = await crawler.get_key_value_store() + persisted = await kvs.get_value('MY_CRAWLER_STATISTICS') + print(f'Persisted finished requests: {persisted["requestsFinished"]}') + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code_examples/statistics/read_final_statistics.py b/docs/02_concepts/code_examples/statistics/read_final_statistics.py new file mode 100644 index 0000000000..457edad005 --- /dev/null +++ b/docs/02_concepts/code_examples/statistics/read_final_statistics.py @@ -0,0 +1,34 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler, ParselCrawlingContext + + +async def main() -> None: + crawler = ParselCrawler(max_requests_per_crawl=10) + + @crawler.router.default_handler + async def request_handler(context: ParselCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url} ...') + await context.enqueue_links() + + # `run()` returns a snapshot of the statistics at the end of the run. + final_stats = await crawler.run(['https://crawlee.dev']) + + print(f'Finished requests: {final_stats.requests_finished}') + print(f'Failed requests: {final_stats.requests_failed}') + print(f'Average duration: {final_stats.request_avg_finished_duration}') + print(f'Crawler runtime: {final_stats.crawler_runtime}') + + # The full state, including fields that `FinalStatistics` doesn't expose. + state = crawler.statistics.state + print(f'Status codes: {state.requests_with_status_code}') + print(f'Slowest request: {state.request_max_duration}') + + # Errors grouped by similarity. + error_tracker = crawler.statistics.error_tracker + print(f'Total errors: {error_tracker.total}') + print(f'Most common errors: {error_tracker.get_most_common_errors()}') + + +if __name__ == '__main__': + asyncio.run(main()) From 1ac948564e1b4333f66a8f24379714f121c0d4af Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 12:31:18 +0200 Subject: [PATCH 06/11] docs: correct how crawler-level services are registered --- docs/02_concepts/25_service_locator.mdx | 2 +- .../service_locator/service_crawler_configuration.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/docs/02_concepts/25_service_locator.mdx b/docs/02_concepts/25_service_locator.mdx index 1f3d785111..6017fab199 100644 --- a/docs/02_concepts/25_service_locator.mdx +++ b/docs/02_concepts/25_service_locator.mdx @@ -77,7 +77,7 @@ Services can be registered globally through the `ServiceLocator` under the hood, making them available to all components and reaching consistent configuration. +Alternatively, services can be passed to the crawler constructors. The crawler keeps them in a service locator of its own, so they apply to that crawler and leave the global `ServiceLocator` untouched. Some components always read the global services, for example the log level and the browser launch defaults. For details, see [Configuration](./configuration#applying-a-configuration-to-one-crawler). diff --git a/docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py b/docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py index 50b13fee71..c0e360c1a5 100644 --- a/docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py +++ b/docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py @@ -1,5 +1,4 @@ import asyncio -from datetime import timedelta from crawlee.configuration import Configuration from crawlee.crawlers import ParselCrawler @@ -7,9 +6,7 @@ async def main() -> None: configuration = Configuration( - log_level='DEBUG', - headless=False, - persist_state_interval=timedelta(seconds=30), + purge_on_start=False, ) # Register configuration via crawler. From b4341c2aa2d418f446eb933609854815edbbd8f5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 12:31:25 +0200 Subject: [PATCH 07/11] docs: link existing pages to the new concept pages --- docs/02_concepts/04_request_router.mdx | 4 ++-- docs/02_concepts/11_storages.mdx | 6 +++--- docs/02_concepts/18_scaling_crawlers.mdx | 2 +- docs/02_concepts/20_error_handling.mdx | 2 +- docs/02_concepts/23_logging.mdx | 4 ++-- docs/02_concepts/25_service_locator.mdx | 4 ++-- docs/03_guides/06_stopping_and_resuming.mdx | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/02_concepts/04_request_router.mdx b/docs/02_concepts/04_request_router.mdx index 204a04da25..01921c2309 100644 --- a/docs/02_concepts/04_request_router.mdx +++ b/docs/02_concepts/04_request_router.mdx @@ -22,7 +22,7 @@ The `Router` class manages request flow and ## Request handlers -Request handlers are user-defined functions that process individual requests and their corresponding responses. Each handler receives a crawling context as its primary argument, which provides access to the current request, response data, and utility methods for data extraction, link enqueuing, and storage operations. Handlers determine how different types of pages are processed and how data is extracted and stored. +Request handlers are user-defined functions that process individual requests and their corresponding responses. Each handler receives a crawling context as its primary argument, which provides access to the current request, response data, and utility methods for data extraction, link enqueuing, and storage operations. Handlers determine how different types of pages are processed and how data is extracted and stored. For the attributes and helpers available on the context, see [Crawling context](./crawling-context). :::note @@ -38,7 +38,7 @@ Every crawler instance includes a built-in `Router` -The default handler processes all requests that either lack a label or have a label for which no specific handler has been registered. +The default handler processes all requests that either lack a label or have a label for which no specific handler has been registered. Labels are stored on the request. For how to set them and pass data between handlers with `user_data`, see [Requests](./requests#labels-and-user-data). ### Custom router diff --git a/docs/02_concepts/11_storages.mdx b/docs/02_concepts/11_storages.mdx index 9157d3cb4f..0bfa3aec6b 100644 --- a/docs/02_concepts/11_storages.mdx +++ b/docs/02_concepts/11_storages.mdx @@ -95,7 +95,7 @@ Each storage type (`Dataset`, `RequestQueue` is the primary storage for URLs in Crawlee, especially useful for deep crawling. It supports dynamic addition of URLs, making it ideal for recursive tasks where URLs are discovered and added during the crawling process (e.g., following links across multiple pages). Each Crawlee project has a **default request queue**, which can be used to store URLs during a specific run. +The `RequestQueue` is the primary storage for URLs in Crawlee, especially useful for deep crawling. It supports dynamic addition of URLs, making it ideal for recursive tasks where URLs are discovered and added during the crawling process (e.g., following links across multiple pages). Each Crawlee project has a **default request queue**, which can be used to store URLs during a specific run. For how the queue deduplicates requests, see [Requests](./requests#unique-key-and-deduplication). The following code demonstrates the usage of the `RequestQueue`: @@ -119,7 +119,7 @@ The following code demonstrates the usage of the `RequestQueue`: +Crawlee provides helper functions to simplify interactions with the `RequestQueue`: Requests and data added through the context helpers are written only after the request handler succeeds. For details, see [Crawling context](./crawling-context#results-are-committed-after-the-handler-succeeds). - The `add_requests` function allows you to manually add specific URLs to the configured request storage. In this case, you must explicitly provide the URLs you want to be added to the request storage. If you need to specify further details of the request, such as a `label` or `user_data`, you have to pass instances of the `Request` class to the helper. - The `enqueue_links` function is designed to discover new URLs in the current page and add them to the request storage. It can be used with default settings, requiring no arguments, or you can customize its behavior by specifying link element selectors, choosing different enqueue strategies, or applying include/exclude filters to control which URLs are added. See the [Enqueue strategies](../guides/crawling-links#enqueue-strategies) section of the Crawling links guide for more details. @@ -246,7 +246,7 @@ The cleanup occurs as soon as a storage is accessed: ### Disabling automatic purging -To disable automatic purging, set `purge_on_start=False` in your configuration: +To disable automatic purging, set `purge_on_start=False` in your configuration: For setting `purge_on_start` and `storage_dir` in code or through environment variables, see [Configuration](./configuration). {CleaningDoNotPurgeExample} diff --git a/docs/02_concepts/18_scaling_crawlers.mdx b/docs/02_concepts/18_scaling_crawlers.mdx index cad2e5fef4..0e43ad39a7 100644 --- a/docs/02_concepts/18_scaling_crawlers.mdx +++ b/docs/02_concepts/18_scaling_crawlers.mdx @@ -50,6 +50,6 @@ The `AutoscaledPool` manages a pool ## Running under a resource limit -A crawler often gets less than the host machine has. A Docker container, a Kubernetes pod, a systemd slice and a Windows job object can each carry a limit of their own. Crawlee reads the limit that applies to the process and scales against it, with nothing to configure. The memory budget comes from the memory limit, the CPU load is measured against the cores the process may use, and the tightest limit wins when several of them apply. Without a limit, Crawlee falls back to the resources of the host machine. +A crawler often gets less than the host machine has. A Docker container, a Kubernetes pod, a systemd slice and a Windows job object can each carry a limit of their own. Crawlee reads the limit that applies to the process and scales against it, with nothing to configure. The memory budget comes from the memory limit, the CPU load is measured against the cores the process may use, and the tightest limit wins when several of them apply. Without a limit, Crawlee falls back to the resources of the host machine. For the environment variables behind these options, see [Configuration](./configuration). The budget is `available_memory_ratio` of the limit, 25% by default, and the crawler throttles once it uses `max_used_memory_ratio` of that budget, 90% by default. A container limited to 2 GB therefore throttles at around 460 MB. Both options live in the `Configuration`, together with `memory_mbytes` for sizing the budget in absolute terms. While the default ratio applies under a limit, Crawlee logs the resulting budget as a warning. Setting either `available_memory_ratio` or `memory_mbytes` silences the warning. Whatever the budget, the crawler also throttles once the memory charged against the limit goes above 97% of it, which includes memory used by other processes under the same limit. diff --git a/docs/02_concepts/20_error_handling.mdx b/docs/02_concepts/20_error_handling.mdx index 99d480d0ea..d5abc3a998 100644 --- a/docs/02_concepts/20_error_handling.mdx +++ b/docs/02_concepts/20_error_handling.mdx @@ -15,7 +15,7 @@ import DisableRetry from '!!raw-loader!roa-loader!./code_examples/error_handling import ParselCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/error_handling/parsel_crawler_with_error_snapshotter.py'; import PlaywrightCrawlerWithErrorSnapshotter from '!!raw-loader!roa-loader!./code_examples/error_handling/playwright_crawler_with_error_snapshotter.py'; -This page demonstrates how Crawlee deals with errors during crawling and the techniques available for handling them. +This page demonstrates how Crawlee deals with errors during crawling and the techniques available for handling them. For how retries are counted and how to turn them off for a single request, see [Requests](./requests#retries). ## Handling proxy errors diff --git a/docs/02_concepts/23_logging.mdx b/docs/02_concepts/23_logging.mdx index 22fae255ea..d1647cab09 100644 --- a/docs/02_concepts/23_logging.mdx +++ b/docs/02_concepts/23_logging.mdx @@ -15,11 +15,11 @@ Crawlee builds on the standard Python [`logging`](https://docs.python.org/3/libr By default, a crawler sets up the logging infrastructure automatically when it's created. It attaches a handler with a human-readable, colored formatter to the root logger, unless the root logger already has handlers configured. You can opt out by passing `configure_logging=False` to the crawler's constructor (see `BasicCrawlerOptions`) and configure logging yourself. -The log level is controlled by the `Configuration.log_level` option, which can also be set through the `CRAWLEE_LOG_LEVEL` environment variable. +The log level is controlled by the `Configuration.log_level` option, which can also be set through the `CRAWLEE_LOG_LEVEL` environment variable. For the other settings and the environment variables behind them, see [Configuration](./configuration). ## Statistics logs -While running, the crawler periodically logs its statistics - requests finished and failed, retry histogram, durations, and more - and prints a final summary when the crawl ends. The `statistics_log_format` option of the crawler's constructor controls the output format: `table` (the default) renders the statistics as a formatted table, while `inline` logs them as a plain message with the values attached as extra fields, which is easier to consume with external log tooling. +While running, the crawler periodically logs its statistics - requests finished and failed, retry histogram, durations, and more - and prints a final summary when the crawl ends. The `statistics_log_format` option of the crawler's constructor controls the output format: `table` (the default) renders the statistics as a formatted table, while `inline` logs them as a plain message with the values attached as extra fields, which is easier to consume with external log tooling. To change the logging interval or persist the statistics, see [Statistics](./statistics). ## JSON logging with loguru diff --git a/docs/02_concepts/25_service_locator.mdx b/docs/02_concepts/25_service_locator.mdx index 6017fab199..c53357cd7c 100644 --- a/docs/02_concepts/25_service_locator.mdx +++ b/docs/02_concepts/25_service_locator.mdx @@ -33,7 +33,7 @@ There are three core services that are managed by the service locator: ### Configuration -`Configuration` is a class that provides access to application-wide settings and parameters. It allows you to configure various aspects of Crawlee, such as timeouts, logging level, persistence intervals, and various other settings. The configuration can be set directly in the code or via environment variables. +`Configuration` is a class that provides access to application-wide settings and parameters. It allows you to configure various aspects of Crawlee, such as timeouts, logging level, persistence intervals, and various other settings. The configuration can be set directly in the code or via environment variables. For the available options and how they resolve, see [Configuration](./configuration). ### StorageClient @@ -43,7 +43,7 @@ Refer to the [Storage clients](./storage-clients) page for more information abou ### EventManager -`EventManager` is responsible for coordinating internal events in Crawlee. It allows you to register event listeners and emit events throughout the framework. Examples of such events aborting, migrating, system info, or browser-specific events like page created, page closed and more. It provides a way to listen to events and execute custom logic when certain events occur. +`EventManager` is responsible for coordinating internal events in Crawlee. It allows you to register event listeners and emit events throughout the framework. Examples of such events aborting, migrating, system info, or browser-specific events like page created, page closed and more. It provides a way to listen to events and execute custom logic when certain events occur. For the events and how to listen to them, see [Events and state persistence](./events). ## Service registration diff --git a/docs/03_guides/06_stopping_and_resuming.mdx b/docs/03_guides/06_stopping_and_resuming.mdx index 2a05f28420..12582d720b 100644 --- a/docs/03_guides/06_stopping_and_resuming.mdx +++ b/docs/03_guides/06_stopping_and_resuming.mdx @@ -33,7 +33,7 @@ To keep the crawler running even when there are no requests to process at the mo ## Resuming a paused crawl -When a local crawler run is interrupted - say with `CTRL+C` - the default behavior is to start over on the next run, because Crawlee purges the storage on start. To continue from the previous state instead, disable `purge_on_start` in the `Configuration`. +When a local crawler run is interrupted - say with `CTRL+C` - the default behavior is to start over on the next run, because Crawlee purges the storage on start. To continue from the previous state instead, disable `purge_on_start` in the `Configuration`. For what Crawlee persists and when, see [Events and state persistence](../concepts/events#state-persistence). Use the following code and perform two sequential runs. During the first run, stop the crawler by pressing `CTRL+C`, and the second run will resume crawling from where it stopped. From 74218c16e90da4c5fa1b87970864b899eb43668f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 12:37:45 +0200 Subject: [PATCH 08/11] docs: keep the existing sidebar categories --- README.md | 2 +- docs/01_introduction/index.mdx | 61 ------------------- .../beautifulsoup_crawler_example.py | 0 .../code_examples}/parsel_crawler_example.py | 0 .../playwright_crawler_example.py | 0 .../playwright_crawler_headful_example.py | 0 .../index.mdx} | 13 ++-- .../01_setting_up.mdx | 0 .../02_first_crawler.mdx | 0 .../03_adding_more_urls.mdx | 0 .../04_real_world_project.mdx | 0 .../05_crawling.mdx | 0 .../06_scraping.mdx | 0 .../07_saving_data.mdx | 0 .../08_refactoring.mdx | 2 +- .../09_running_in_cloud.mdx | 2 +- .../code_examples/02_bs.py | 0 .../code_examples/02_bs_better.py | 0 .../code_examples/02_request_queue.py | 0 .../code_examples/03_enqueue_strategy.py | 0 .../code_examples/03_finding_new_links.py | 0 .../code_examples/03_globs.py | 0 .../code_examples/03_original_code.py | 0 .../code_examples/03_transform_request.py | 0 .../code_examples/04_sanity_check.py | 0 .../code_examples/05_crawling_detail.py | 0 .../code_examples/05_crawling_listing.py | 0 .../code_examples/06_scraping.py | 0 .../code_examples/07_final_code.py | 0 .../code_examples/07_first_code.py | 0 .../code_examples/08_main.py | 0 .../code_examples/08_routes.py | 0 .../code_examples/09_apify_sdk.py | 0 .../code_examples/__init__.py | 0 .../code_examples/routes.py | 0 docs/02_introduction/index.mdx | 56 +++++++++++++++++ .../01_architecture_overview.mdx | 0 .../02_requests.mdx | 0 .../03_crawling_context.mdx | 0 .../04_request_router.mdx | 0 .../05_http_crawlers.mdx | 0 .../06_playwright_crawler.mdx | 0 .../07_playwright_crawler_adaptive.mdx | 0 .../08_pydantic_ai_crawler.mdx | 0 .../09_stagehand_crawler.mdx | 0 .../10_request_loaders.mdx | 0 .../11_storages.mdx | 0 .../12_storage_clients.mdx | 0 .../13_http_clients.mdx | 0 .../14_session_management.mdx | 0 .../15_cookie_management.mdx | 0 .../16_http_headers.mdx | 0 .../17_proxy_management.mdx | 0 .../18_scaling_crawlers.mdx | 0 .../19_request_throttling.mdx | 0 .../20_error_handling.mdx | 0 .../21_events.mdx | 0 .../22_statistics.mdx | 0 .../23_logging.mdx | 0 .../24_configuration.mdx | 0 .../25_service_locator.mdx | 0 .../26_extending_crawlee.mdx | 0 .../configuration/crawler_configuration.py | 0 .../configuration/global_configuration.py | 0 .../cookie_management/initial_cookies.py | 0 .../cookie_management/persist_cookies.py | 0 .../cookie_management/playwright_cookies.py | 0 .../cookie_management/read_write_cookies.py | 0 .../cookie_management/retry_pinned_session.py | 0 .../retry_restore_cookies.py | 0 .../cookie_management/retry_single_session.py | 0 .../crawling_context/buffered_results.py | 0 .../crawling_context/context_helpers.py | 0 .../change_handle_error_status.py | 0 .../error_handling/disable_retry.py | 0 .../error_handling/handle_proxy_error.py | 0 .../parsel_crawler_with_error_snapshotter.py | 0 ...aywright_crawler_with_error_snapshotter.py | 0 .../code_examples/events/listen_to_events.py | 0 .../code_examples/events/use_state.py | 0 .../parsel_curl_impersonate_example.py | 0 .../http_clients/parsel_httpx_example.py | 0 .../http_clients/parsel_impit_example.py | 0 .../code_examples/http_crawlers/__init__.py | 0 .../http_crawlers/beautifulsoup_example.py | 0 .../http_crawlers/custom_crawler_example.py | 0 .../http_crawlers/file_download.py | 0 .../http_crawlers/file_download_stream.py | 0 .../http_crawlers/http_example.py | 0 .../http_crawlers/lexbor_parser.py | 0 .../http_crawlers/lxml_parser.py | 0 .../http_crawlers/lxml_saxonche_parser.py | 0 .../http_crawlers/parsel_example.py | 0 .../http_crawlers/pyquery_parser.py | 0 .../http_crawlers/scrapling_parser.py | 0 .../http_crawlers/selectolax_adaptive_run.py | 0 .../http_crawlers/selectolax_context.py | 0 .../http_crawlers/selectolax_crawler.py | 0 .../http_crawlers/selectolax_crawler_run.py | 0 .../http_crawlers/selectolax_parser.py | 0 .../http_headers/browser_page_headers.py | 0 .../code_examples/http_headers/set_headers.py | 0 .../logging/configure_json_logging.py | 0 .../browser_configuration_example.py | 0 .../browser_pool_launch_hooks_example.py | 0 .../browser_pool_page_hooks_example.py | 0 .../capture_screenshot_using_playwright.py | 0 .../multiple_launch_example.py | 0 .../navigation_hooks_example.py | 0 .../playwright_block_requests.py | 0 .../playwright_crawler/playwright_crawler.py | 0 .../plugin_browser_configuration_example.py | 0 .../adaptive_playwright_crawler.py | 0 .../playwright_crawler_adaptive/handler.py | 0 .../init_beautifulsoup.py | 0 .../init_parsel.py | 0 .../init_prediction.py | 0 .../pre_nav_hooks.py | 0 .../proxy_management/inspecting_bs_example.py | 0 .../proxy_management/inspecting_pw_example.py | 0 .../integration_bs_example.py | 0 .../integration_pw_example.py | 0 .../proxy_management/quick_start_example.py | 0 .../proxy_management/session_bs_example.py | 0 .../proxy_management/session_pw_example.py | 0 .../proxy_management/tiers_bs_example.py | 0 .../proxy_management/tiers_pw_example.py | 0 .../additional_instructions_example.py | 0 .../pydantic_ai_crawler/basic_example.py | 0 .../custom_distiller_example.py | 0 .../pydantic_ai_crawler/debugging_example.py | 0 .../selector_extractor_example.py | 0 .../usage_limit_example.py | 0 .../request_loaders/rl_basic_example.py | 0 .../rl_basic_example_with_persist.py | 0 .../request_loaders/rl_tandem_example.py | 0 .../rl_tandem_example_explicit.py | 0 .../request_loaders/sitemap_basic_example.py | 0 .../sitemap_example_with_persist.py | 0 .../request_loaders/sitemap_tandem_example.py | 0 .../sitemap_tandem_example_explicit.py | 0 .../using_sitemap_request_loader.py | 0 .../adaptive_crawler_handlers.py | 0 .../request_router/basic_request_handlers.py | 0 .../custom_router_default_only.py | 0 .../request_router/error_handler.py | 0 .../request_router/failed_request_handler.py | 0 .../request_router/http_pre_navigation.py | 0 .../playwright_pre_navigation.py | 0 .../request_router/router_middleware.py | 0 .../request_router/simple_default_handler.py | 0 .../request_throttling/throttling_example.py | 0 .../requests/label_and_user_data.py | 0 .../requests/unique_key_deduplication.py | 0 .../max_tasks_per_minute_example.py | 0 .../min_and_max_concurrency_example.py | 0 .../service_locator/service_conflicts.py | 0 .../service_crawler_configuration.py | 0 .../service_crawler_event_manager.py | 0 .../service_crawler_storage_client.py | 0 .../service_locator_configuration.py | 0 .../service_locator_event_manager.py | 0 .../service_locator_storage_client.py | 0 .../service_storage_configuration.py | 0 .../service_storage_storage_client.py | 0 .../session_management/multi_sessions_http.py | 0 .../session_management/one_session_http.py | 0 .../session_management/sm_basic.py | 0 .../session_management/sm_beautifulsoup.py | 0 .../session_management/sm_http.py | 0 .../session_management/sm_parsel.py | 0 .../session_management/sm_playwright.py | 0 .../session_management/sm_standalone.py | 0 .../stagehand_crawler/basic_example.py | 0 .../stagehand_crawler/browserbase_example.py | 0 .../statistics/custom_statistics.py | 0 .../statistics/read_final_statistics.py | 0 .../custom_storage_client_example.py | 0 ...ile_system_storage_client_basic_example.py | 0 ...em_storage_client_configuration_example.py | 0 .../memory_storage_client_basic_example.py | 0 .../redis_storage_client_basic_example.py | 0 ...is_storage_client_configuration_example.py | 0 .../registering_storage_clients_example.py | 0 .../sql_storage_client_basic_example.py | 0 ...ql_storage_client_configuration_example.py | 0 .../storages/cleaning_do_not_purge_example.py | 0 .../cleaning_purge_explicitly_example.py | 0 .../storages/dataset_basic_example.py | 0 .../storages/dataset_with_crawler_example.py | 0 .../dataset_with_crawler_explicit_example.py | 0 .../export_entire_dataset_to_file_csv.py | 0 .../export_entire_dataset_to_file_json.py | 0 .../storages/helper_add_requests_example.py | 0 .../storages/helper_enqueue_links_example.py | 0 .../storages/kvs_basic_example.py | 0 .../storages/kvs_with_crawler_example.py | 0 .../kvs_with_crawler_explicit_example.py | 0 .../code_examples/storages/opening.py | 0 .../storages/rq_basic_example.py | 0 .../storages/rq_with_crawler_example.py | 0 .../rq_with_crawler_explicit_example.py | 0 .../01_crawling_links.mdx | 0 .../02_fill_and_submit_web_form.mdx | 0 .../03_crawler_login.mdx | 0 .../04_avoid_blocking.mdx | 0 .../05_respect_robots_txt_file.mdx | 0 .../06_stopping_and_resuming.mdx | 0 .../07_run_parallel_crawlers.mdx | 0 .../08_running_in_web_server.mdx | 0 .../09_trace_and_monitor_crawlers.mdx | 0 .../10_creating_web_archive.mdx | 0 .../11_secure_scraping.mdx | 2 +- .../12_scrapy_migration.mdx | 0 ...default_fingerprint_generator_with_args.py | 0 .../playwright_crawler_with_camoufox.py | 0 .../playwright_with_cloakbrowser.py | 0 .../playwright_with_fingerprint_generator.py | 0 .../crawl_all_links_on_website_bs.py | 0 .../crawl_all_links_on_website_pw.py | 0 .../crawling_links/crawl_multiple_urls_bs.py | 0 .../crawling_links/crawl_multiple_urls_pw.py | 0 .../crawl_specific_links_on_website_bs.py | 0 .../crawl_specific_links_on_website_pw.py | 0 ...l_website_with_relative_links_all_links.py | 0 ...website_with_relative_links_same_domain.py | 0 ...bsite_with_relative_links_same_hostname.py | 0 ...website_with_relative_links_same_origin.py | 0 ...ct_and_add_specific_links_on_website_bs.py | 0 ...ct_and_add_specific_links_on_website_pw.py | 0 .../manual_archiving_parsel_crawler.py | 0 .../manual_archiving_playwright_crawler.py | 0 .../simple_pw_through_proxy_pywb_server.py | 0 .../fill_and_submit_web_form_automated.py | 0 .../fill_and_submit_web_form_crawler.py | 0 .../fill_and_submit_web_form_request.py | 0 .../code_examples/login_crawler/http_login.py | 0 .../login_crawler/playwright_login.py | 0 .../using_browser_profiles_chrome.py | 0 .../using_browser_profiles_firefox.py | 0 .../respect_robots_on_skipped_request.py | 0 .../respect_robots_txt_file.py | 0 .../run_parallel_crawlers.py | 0 .../running_in_web_server/__init__.py | 0 .../running_in_web_server/crawler.py | 0 .../running_in_web_server/server.py | 0 .../scrapy_migration/crawlee_concurrency.py | 0 .../scrapy_migration/crawlee_crawlspider.py | 0 .../crawlee_error_handling.py | 0 .../scrapy_migration/crawlee_export.py | 0 .../scrapy_migration/crawlee_labels.py | 0 .../scrapy_migration/crawlee_playwright.py | 0 .../scrapy_migration/crawlee_post.py | 0 .../scrapy_migration/crawlee_proxy.py | 0 .../scrapy_migration/crawlee_quotes.py | 0 .../scrapy_migration/crawlee_throttling.py | 0 .../scrapy_migration/crawlee_user_data.py | 0 .../scrapy_migration/scrapy_authors.py | 0 .../scrapy_migration/scrapy_cb_kwargs.py | 0 .../scrapy_migration/scrapy_concurrency.py | 0 .../scrapy_migration/scrapy_crawlspider.py | 0 .../scrapy_migration/scrapy_errback.py | 0 .../scrapy_migration/scrapy_export.py | 0 .../scrapy_migration/scrapy_formrequest.py | 0 .../scrapy_migration/scrapy_playwright.py | 0 .../scrapy_playwright_settings.py | 0 .../scrapy_migration/scrapy_proxy.py | 0 .../scrapy_migration/scrapy_quotes.py | 0 .../scrapy_migration/scrapy_throttling.py | 0 .../beautifulsoup_crawler_keep_alive.py | 0 .../beautifulsoup_crawler_stop.py | 0 .../resuming_paused_crawl.py | 0 .../instrument_crawler.py | 0 .../apify_platform.mdx} | 16 ++--- .../aws_lambda.mdx} | 10 +-- .../apify/crawler_as_actor_example.py | 0 .../code_examples}/apify/get_public_url.py | 0 .../apify/log_with_config_example.py | 0 .../apify/proxy_advanced_example.py | 0 .../code_examples}/apify/proxy_example.py | 0 .../aws/beautifulsoup_crawler_lambda.py | 0 .../aws/playwright_crawler_lambda.py | 0 .../code_examples}/aws/playwright_dockerfile | 0 .../google/cloud_run_example.py | 0 .../code_examples}/google/google_example.py | 0 .../google_cloud.mdx} | 6 +- .../google_cloud_run.mdx} | 8 +-- .../upgrading_to_v0x.md | 0 .../upgrading_to_v1.md | 0 pyproject.toml | 10 +-- website/docusaurus.config.js | 18 +++--- website/sidebars.js | 57 +++++++++++------ 292 files changed, 136 insertions(+), 127 deletions(-) delete mode 100644 docs/01_introduction/index.mdx rename docs/{01_introduction/code_examples/quick_start => 01_quick-start/code_examples}/beautifulsoup_crawler_example.py (100%) rename docs/{01_introduction/code_examples/quick_start => 01_quick-start/code_examples}/parsel_crawler_example.py (100%) rename docs/{01_introduction/code_examples/quick_start => 01_quick-start/code_examples}/playwright_crawler_example.py (100%) rename docs/{01_introduction/code_examples/quick_start => 01_quick-start/code_examples}/playwright_crawler_headful_example.py (100%) rename docs/{01_introduction/quick-start.mdx => 01_quick-start/index.mdx} (93%) rename docs/{01_introduction => 02_introduction}/01_setting_up.mdx (100%) rename docs/{01_introduction => 02_introduction}/02_first_crawler.mdx (100%) rename docs/{01_introduction => 02_introduction}/03_adding_more_urls.mdx (100%) rename docs/{01_introduction => 02_introduction}/04_real_world_project.mdx (100%) rename docs/{01_introduction => 02_introduction}/05_crawling.mdx (100%) rename docs/{01_introduction => 02_introduction}/06_scraping.mdx (100%) rename docs/{01_introduction => 02_introduction}/07_saving_data.mdx (100%) rename docs/{01_introduction => 02_introduction}/08_refactoring.mdx (97%) rename docs/{01_introduction => 02_introduction}/09_running_in_cloud.mdx (97%) rename docs/{01_introduction => 02_introduction}/code_examples/02_bs.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/02_bs_better.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/02_request_queue.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/03_enqueue_strategy.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/03_finding_new_links.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/03_globs.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/03_original_code.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/03_transform_request.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/04_sanity_check.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/05_crawling_detail.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/05_crawling_listing.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/06_scraping.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/07_final_code.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/07_first_code.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/08_main.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/08_routes.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/09_apify_sdk.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/__init__.py (100%) rename docs/{01_introduction => 02_introduction}/code_examples/routes.py (100%) create mode 100644 docs/02_introduction/index.mdx rename docs/{02_concepts => 03_concepts}/01_architecture_overview.mdx (100%) rename docs/{02_concepts => 03_concepts}/02_requests.mdx (100%) rename docs/{02_concepts => 03_concepts}/03_crawling_context.mdx (100%) rename docs/{02_concepts => 03_concepts}/04_request_router.mdx (100%) rename docs/{02_concepts => 03_concepts}/05_http_crawlers.mdx (100%) rename docs/{02_concepts => 03_concepts}/06_playwright_crawler.mdx (100%) rename docs/{02_concepts => 03_concepts}/07_playwright_crawler_adaptive.mdx (100%) rename docs/{02_concepts => 03_concepts}/08_pydantic_ai_crawler.mdx (100%) rename docs/{02_concepts => 03_concepts}/09_stagehand_crawler.mdx (100%) rename docs/{02_concepts => 03_concepts}/10_request_loaders.mdx (100%) rename docs/{02_concepts => 03_concepts}/11_storages.mdx (100%) rename docs/{02_concepts => 03_concepts}/12_storage_clients.mdx (100%) rename docs/{02_concepts => 03_concepts}/13_http_clients.mdx (100%) rename docs/{02_concepts => 03_concepts}/14_session_management.mdx (100%) rename docs/{02_concepts => 03_concepts}/15_cookie_management.mdx (100%) rename docs/{02_concepts => 03_concepts}/16_http_headers.mdx (100%) rename docs/{02_concepts => 03_concepts}/17_proxy_management.mdx (100%) rename docs/{02_concepts => 03_concepts}/18_scaling_crawlers.mdx (100%) rename docs/{02_concepts => 03_concepts}/19_request_throttling.mdx (100%) rename docs/{02_concepts => 03_concepts}/20_error_handling.mdx (100%) rename docs/{02_concepts => 03_concepts}/21_events.mdx (100%) rename docs/{02_concepts => 03_concepts}/22_statistics.mdx (100%) rename docs/{02_concepts => 03_concepts}/23_logging.mdx (100%) rename docs/{02_concepts => 03_concepts}/24_configuration.mdx (100%) rename docs/{02_concepts => 03_concepts}/25_service_locator.mdx (100%) rename docs/{02_concepts => 03_concepts}/26_extending_crawlee.mdx (100%) rename docs/{02_concepts => 03_concepts}/code_examples/configuration/crawler_configuration.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/configuration/global_configuration.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/initial_cookies.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/persist_cookies.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/playwright_cookies.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/read_write_cookies.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/retry_pinned_session.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/retry_restore_cookies.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/cookie_management/retry_single_session.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/crawling_context/buffered_results.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/crawling_context/context_helpers.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/error_handling/change_handle_error_status.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/error_handling/disable_retry.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/error_handling/handle_proxy_error.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/events/listen_to_events.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/events/use_state.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_clients/parsel_curl_impersonate_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_clients/parsel_httpx_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_clients/parsel_impit_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/__init__.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/beautifulsoup_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/custom_crawler_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/file_download.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/file_download_stream.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/http_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/lexbor_parser.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/lxml_parser.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/lxml_saxonche_parser.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/parsel_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/pyquery_parser.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/scrapling_parser.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/selectolax_adaptive_run.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/selectolax_context.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/selectolax_crawler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/selectolax_crawler_run.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_crawlers/selectolax_parser.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_headers/browser_page_headers.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/http_headers/set_headers.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/logging/configure_json_logging.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/browser_configuration_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/browser_pool_page_hooks_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/capture_screenshot_using_playwright.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/multiple_launch_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/navigation_hooks_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/playwright_block_requests.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/playwright_crawler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler/plugin_browser_configuration_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler_adaptive/handler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler_adaptive/init_parsel.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler_adaptive/init_prediction.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/inspecting_bs_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/inspecting_pw_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/integration_bs_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/integration_pw_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/quick_start_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/session_bs_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/session_pw_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/tiers_bs_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/proxy_management/tiers_pw_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/pydantic_ai_crawler/additional_instructions_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/pydantic_ai_crawler/basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/pydantic_ai_crawler/custom_distiller_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/pydantic_ai_crawler/debugging_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/pydantic_ai_crawler/selector_extractor_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/pydantic_ai_crawler/usage_limit_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/rl_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/rl_basic_example_with_persist.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/rl_tandem_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/rl_tandem_example_explicit.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/sitemap_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/sitemap_example_with_persist.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/sitemap_tandem_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/sitemap_tandem_example_explicit.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_loaders/using_sitemap_request_loader.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/adaptive_crawler_handlers.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/basic_request_handlers.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/custom_router_default_only.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/error_handler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/failed_request_handler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/http_pre_navigation.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/playwright_pre_navigation.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/router_middleware.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_router/simple_default_handler.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/request_throttling/throttling_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/requests/label_and_user_data.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/requests/unique_key_deduplication.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/scaling_crawlers/max_tasks_per_minute_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/scaling_crawlers/min_and_max_concurrency_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_conflicts.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_crawler_configuration.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_crawler_event_manager.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_crawler_storage_client.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_locator_configuration.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_locator_event_manager.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_locator_storage_client.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_storage_configuration.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/service_locator/service_storage_storage_client.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/multi_sessions_http.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/one_session_http.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/sm_basic.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/sm_beautifulsoup.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/sm_http.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/sm_parsel.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/sm_playwright.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/session_management/sm_standalone.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/stagehand_crawler/basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/stagehand_crawler/browserbase_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/statistics/custom_statistics.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/statistics/read_final_statistics.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/custom_storage_client_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/file_system_storage_client_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/file_system_storage_client_configuration_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/memory_storage_client_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/redis_storage_client_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/redis_storage_client_configuration_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/registering_storage_clients_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/sql_storage_client_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storage_clients/sql_storage_client_configuration_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/cleaning_do_not_purge_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/cleaning_purge_explicitly_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/dataset_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/dataset_with_crawler_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/dataset_with_crawler_explicit_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/export_entire_dataset_to_file_csv.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/export_entire_dataset_to_file_json.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/helper_add_requests_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/helper_enqueue_links_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/kvs_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/kvs_with_crawler_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/kvs_with_crawler_explicit_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/opening.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/rq_basic_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/rq_with_crawler_example.py (100%) rename docs/{02_concepts => 03_concepts}/code_examples/storages/rq_with_crawler_explicit_example.py (100%) rename docs/{03_guides => 04_guides}/01_crawling_links.mdx (100%) rename docs/{03_guides => 04_guides}/02_fill_and_submit_web_form.mdx (100%) rename docs/{03_guides => 04_guides}/03_crawler_login.mdx (100%) rename docs/{03_guides => 04_guides}/04_avoid_blocking.mdx (100%) rename docs/{03_guides => 04_guides}/05_respect_robots_txt_file.mdx (100%) rename docs/{03_guides => 04_guides}/06_stopping_and_resuming.mdx (100%) rename docs/{03_guides => 04_guides}/07_run_parallel_crawlers.mdx (100%) rename docs/{03_guides => 04_guides}/08_running_in_web_server.mdx (100%) rename docs/{03_guides => 04_guides}/09_trace_and_monitor_crawlers.mdx (100%) rename docs/{03_guides => 04_guides}/10_creating_web_archive.mdx (100%) rename docs/{03_guides => 04_guides}/11_secure_scraping.mdx (99%) rename docs/{03_guides => 04_guides}/12_scrapy_migration.mdx (100%) rename docs/{03_guides => 04_guides}/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py (100%) rename docs/{03_guides => 04_guides}/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py (100%) rename docs/{03_guides => 04_guides}/code_examples/avoid_blocking/playwright_with_cloakbrowser.py (100%) rename docs/{03_guides => 04_guides}/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_all_links_on_website_bs.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_all_links_on_website_pw.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_multiple_urls_bs.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_multiple_urls_pw.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_specific_links_on_website_bs.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_specific_links_on_website_pw.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py (100%) rename docs/{03_guides => 04_guides}/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py (100%) rename docs/{03_guides => 04_guides}/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py (100%) rename docs/{03_guides => 04_guides}/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py (100%) rename docs/{03_guides => 04_guides}/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py (100%) rename docs/{03_guides => 04_guides}/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py (100%) rename docs/{03_guides => 04_guides}/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py (100%) rename docs/{03_guides => 04_guides}/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py (100%) rename docs/{03_guides => 04_guides}/code_examples/login_crawler/http_login.py (100%) rename docs/{03_guides => 04_guides}/code_examples/login_crawler/playwright_login.py (100%) rename docs/{03_guides => 04_guides}/code_examples/login_crawler/using_browser_profiles_chrome.py (100%) rename docs/{03_guides => 04_guides}/code_examples/login_crawler/using_browser_profiles_firefox.py (100%) rename docs/{03_guides => 04_guides}/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py (100%) rename docs/{03_guides => 04_guides}/code_examples/respect_robots_txt_file/respect_robots_txt_file.py (100%) rename docs/{03_guides => 04_guides}/code_examples/run_parallel_crawlers/run_parallel_crawlers.py (100%) rename docs/{03_guides => 04_guides}/code_examples/running_in_web_server/__init__.py (100%) rename docs/{03_guides => 04_guides}/code_examples/running_in_web_server/crawler.py (100%) rename docs/{03_guides => 04_guides}/code_examples/running_in_web_server/server.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_concurrency.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_crawlspider.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_error_handling.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_export.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_labels.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_playwright.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_post.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_proxy.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_quotes.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_throttling.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/crawlee_user_data.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_authors.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_cb_kwargs.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_concurrency.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_crawlspider.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_errback.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_export.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_formrequest.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_playwright.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_playwright_settings.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_proxy.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_quotes.py (100%) rename docs/{03_guides => 04_guides}/code_examples/scrapy_migration/scrapy_throttling.py (100%) rename docs/{03_guides => 04_guides}/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py (100%) rename docs/{03_guides => 04_guides}/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py (100%) rename docs/{03_guides => 04_guides}/code_examples/stopping_and_resuming/resuming_paused_crawl.py (100%) rename docs/{03_guides => 04_guides}/code_examples/trace_and_monitor_crawlers/instrument_crawler.py (100%) rename docs/{03_guides/13_deploy_on_apify.mdx => 05_deployment/apify_platform.mdx} (95%) rename docs/{03_guides/14_deploy_on_aws_lambda.mdx => 05_deployment/aws_lambda.mdx} (97%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/apify/crawler_as_actor_example.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/apify/get_public_url.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/apify/log_with_config_example.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/apify/proxy_advanced_example.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/apify/proxy_example.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/aws/beautifulsoup_crawler_lambda.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/aws/playwright_crawler_lambda.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/aws/playwright_dockerfile (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/google/cloud_run_example.py (100%) rename docs/{03_guides/code_examples/deployment => 05_deployment/code_examples}/google/google_example.py (100%) rename docs/{03_guides/15_deploy_on_gcp_cloud_run_functions.mdx => 05_deployment/google_cloud.mdx} (92%) rename docs/{03_guides/16_deploy_on_gcp_cloud_run.mdx => 05_deployment/google_cloud_run.mdx} (91%) rename docs/{04_upgrading => 06_upgrading}/upgrading_to_v0x.md (100%) rename docs/{04_upgrading => 06_upgrading}/upgrading_to_v1.md (100%) diff --git a/README.md b/README.md index e9d4a43b64..6e33b33fab 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ We also have a TypeScript implementation of the Crawlee, which you can explore a ## Installation -For a step-by-step walkthrough, see the [tutorial](https://crawlee.dev/python/docs/introduction/tutorial) in the Crawlee documentation. +We recommend visiting the [Introduction tutorial](https://crawlee.dev/python/docs/introduction) in Crawlee documentation for more information. Crawlee is available as [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. This package includes the core functionality, while additional features are available as optional extras to keep dependencies and package size minimal. diff --git a/docs/01_introduction/index.mdx b/docs/01_introduction/index.mdx deleted file mode 100644 index 7d79536567..0000000000 --- a/docs/01_introduction/index.mdx +++ /dev/null @@ -1,61 +0,0 @@ ---- -id: introduction -title: Overview -description: Crawlee for Python is a web scraping and browser automation library with a unified interface for HTTP and headless browser crawling. ---- - -import ApiLink from '@site/src/components/ApiLink'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; - -import OverviewExample from '!!raw-loader!roa-loader!./code_examples/quick_start/beautifulsoup_crawler_example.py'; - -Crawlee for Python is a web scraping and browser automation library. It handles the crawling infrastructure for you: it queues and deduplicates requests, retries failures, scales concurrency to the available system resources, rotates proxies and sessions, and stores the results. You write the code that extracts the data. - -The same interface works for plain HTTP crawling and for headless browsers, so you can switch between them without rewriting your handlers. With the default configuration, the HTTP client impersonates a real browser and browser crawlers use generated fingerprints, which helps them get past common bot protections. - - - {OverviewExample} - - -## Features - -- One interface for HTTP and headless browser crawling. -- Automatic parallel crawling based on available CPU and memory. For details, see [Scaling crawlers](./concepts/scaling-crawlers). -- Automatic retries on errors and when the crawler gets blocked. For details, see [Error handling](./concepts/error-handling). -- Proxy rotation and session management. For details, see [Proxy management](./concepts/proxy-management) and [Session management](./concepts/session-management). -- Request routing that sends each URL to the right handler. For details, see [Request router](./concepts/request-router). -- A persistent request queue and pluggable storages for tabular data and files. For details, see [Storages](./concepts/storages). -- Full type hint coverage, built on standard `asyncio`. - -## Crawler types - -Crawlee provides several crawlers that share the same base class, `BasicCrawler`: - -- `BeautifulSoupCrawler` and `ParselCrawler` fetch pages over HTTP and parse the HTML. They're fast and light, but they can't run JavaScript. For details, see [HTTP crawlers](./concepts/http-crawlers). -- `PlaywrightCrawler` controls a headless browser through [Playwright](https://playwright.dev/), for pages that render their content with JavaScript. For details, see [Playwright crawler](./concepts/playwright-crawler). -- `AdaptivePlaywrightCrawler` decides per request whether a plain HTTP fetch is enough or a browser is needed. For details, see [Adaptive Playwright crawler](./concepts/adaptive-playwright-crawler). -- `PydanticAiCrawler` and `StagehandCrawler` use an LLM to extract data or drive the browser. For details, see [Pydantic AI crawler](./concepts/pydantic-ai-crawler) and [Stagehand crawler](./concepts/stagehand-crawler). - -## Installation - -Crawlee requires Python 3.10 or higher. It's available as the [`crawlee`](https://pypi.org/project/crawlee/) package on PyPI. The core package is small, and each crawler type comes with an optional extra. To install Crawlee with all extras, run: - -```sh -pip install 'crawlee[all]' -``` - -To use the `PlaywrightCrawler`, also install the browser binaries: - -```sh -playwright install -``` - -For the list of extras and for bootstrapping a project from a template with the Crawlee CLI, see [Setting up](./introduction/setting-up). - -## Next steps - -- To run your first crawler in a few minutes, see the [Quick start](./quick-start). -- To build a complete scraper for a real website step by step, follow the [Tutorial](./introduction/tutorial). -- To learn how the parts of Crawlee work, read the [Concepts](./concepts). -- For solutions to specific tasks, such as logging in, filling in forms, or deploying a crawler, see the [Guides](./guides). -- If you're coming from Scrapy, see [Migrating from Scrapy](./guides/scrapy-migration). diff --git a/docs/01_introduction/code_examples/quick_start/beautifulsoup_crawler_example.py b/docs/01_quick-start/code_examples/beautifulsoup_crawler_example.py similarity index 100% rename from docs/01_introduction/code_examples/quick_start/beautifulsoup_crawler_example.py rename to docs/01_quick-start/code_examples/beautifulsoup_crawler_example.py diff --git a/docs/01_introduction/code_examples/quick_start/parsel_crawler_example.py b/docs/01_quick-start/code_examples/parsel_crawler_example.py similarity index 100% rename from docs/01_introduction/code_examples/quick_start/parsel_crawler_example.py rename to docs/01_quick-start/code_examples/parsel_crawler_example.py diff --git a/docs/01_introduction/code_examples/quick_start/playwright_crawler_example.py b/docs/01_quick-start/code_examples/playwright_crawler_example.py similarity index 100% rename from docs/01_introduction/code_examples/quick_start/playwright_crawler_example.py rename to docs/01_quick-start/code_examples/playwright_crawler_example.py diff --git a/docs/01_introduction/code_examples/quick_start/playwright_crawler_headful_example.py b/docs/01_quick-start/code_examples/playwright_crawler_headful_example.py similarity index 100% rename from docs/01_introduction/code_examples/quick_start/playwright_crawler_headful_example.py rename to docs/01_quick-start/code_examples/playwright_crawler_headful_example.py diff --git a/docs/01_introduction/quick-start.mdx b/docs/01_quick-start/index.mdx similarity index 93% rename from docs/01_introduction/quick-start.mdx rename to docs/01_quick-start/index.mdx index 25343b0023..6ba5d8d2b7 100644 --- a/docs/01_introduction/quick-start.mdx +++ b/docs/01_quick-start/index.mdx @@ -1,7 +1,6 @@ --- id: quick-start title: Quick start -slug: /quick-start description: Build and run your first Crawlee crawler in a few minutes, and pick the crawler type that fits your project. --- @@ -11,13 +10,13 @@ import TabItem from '@theme/TabItem'; import CodeBlock from '@theme/CodeBlock'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/quick_start/beautifulsoup_crawler_example.py'; -import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/quick_start/parsel_crawler_example.py'; -import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/quick_start/playwright_crawler_example.py'; +import BeautifulsoupCrawlerExample from '!!raw-loader!roa-loader!./code_examples/beautifulsoup_crawler_example.py'; +import ParselCrawlerExample from '!!raw-loader!roa-loader!./code_examples/parsel_crawler_example.py'; +import PlaywrightCrawlerExample from '!!raw-loader!roa-loader!./code_examples/playwright_crawler_example.py'; -import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/quick_start/playwright_crawler_headful_example.py'; +import PlaywrightCrawlerHeadfulExample from '!!raw-loader!./code_examples/playwright_crawler_headful_example.py'; -This page gets you from installation to a running crawler in a few minutes. To build a complete scraper step by step, follow the [Tutorial](./introduction/tutorial). +This page gets you from installation to a running crawler in a few minutes. To build a complete scraper step by step, follow the [Introduction](./introduction). ## Choose your crawler @@ -130,6 +129,6 @@ To change the storage directory, set the `CRAWLEE_STORAGE_DIR` environment varia ## Next steps -- To build a complete scraper for a real website, follow the [Tutorial](./introduction/tutorial). +- To build a complete scraper for a real website step by step, follow the [Introduction](./introduction). - To learn how crawlers, requests, storages, and other parts of Crawlee work, read the [Concepts](./concepts). - For solutions to specific tasks, see the [Guides](./guides). diff --git a/docs/01_introduction/01_setting_up.mdx b/docs/02_introduction/01_setting_up.mdx similarity index 100% rename from docs/01_introduction/01_setting_up.mdx rename to docs/02_introduction/01_setting_up.mdx diff --git a/docs/01_introduction/02_first_crawler.mdx b/docs/02_introduction/02_first_crawler.mdx similarity index 100% rename from docs/01_introduction/02_first_crawler.mdx rename to docs/02_introduction/02_first_crawler.mdx diff --git a/docs/01_introduction/03_adding_more_urls.mdx b/docs/02_introduction/03_adding_more_urls.mdx similarity index 100% rename from docs/01_introduction/03_adding_more_urls.mdx rename to docs/02_introduction/03_adding_more_urls.mdx diff --git a/docs/01_introduction/04_real_world_project.mdx b/docs/02_introduction/04_real_world_project.mdx similarity index 100% rename from docs/01_introduction/04_real_world_project.mdx rename to docs/02_introduction/04_real_world_project.mdx diff --git a/docs/01_introduction/05_crawling.mdx b/docs/02_introduction/05_crawling.mdx similarity index 100% rename from docs/01_introduction/05_crawling.mdx rename to docs/02_introduction/05_crawling.mdx diff --git a/docs/01_introduction/06_scraping.mdx b/docs/02_introduction/06_scraping.mdx similarity index 100% rename from docs/01_introduction/06_scraping.mdx rename to docs/02_introduction/06_scraping.mdx diff --git a/docs/01_introduction/07_saving_data.mdx b/docs/02_introduction/07_saving_data.mdx similarity index 100% rename from docs/01_introduction/07_saving_data.mdx rename to docs/02_introduction/07_saving_data.mdx diff --git a/docs/01_introduction/08_refactoring.mdx b/docs/02_introduction/08_refactoring.mdx similarity index 97% rename from docs/01_introduction/08_refactoring.mdx rename to docs/02_introduction/08_refactoring.mdx index e5d7597a88..7312ea8f8b 100644 --- a/docs/01_introduction/08_refactoring.mdx +++ b/docs/02_introduction/08_refactoring.mdx @@ -70,4 +70,4 @@ It's good practice in any programming language to split your logic into bite-siz ## Next steps -In the next and final step, you'll see how to deploy your Crawlee project to the cloud. If you used the CLI to bootstrap your project, you already have a `Dockerfile` ready, and the next section will show you how to deploy it to the [Apify platform](../guides/deploy-on-apify) with ease. +In the next and final step, you'll see how to deploy your Crawlee project to the cloud. If you used the CLI to bootstrap your project, you already have a `Dockerfile` ready, and the next section will show you how to deploy it to the [Apify platform](../deployment/apify-platform) with ease. diff --git a/docs/01_introduction/09_running_in_cloud.mdx b/docs/02_introduction/09_running_in_cloud.mdx similarity index 97% rename from docs/01_introduction/09_running_in_cloud.mdx rename to docs/02_introduction/09_running_in_cloud.mdx index d2339a921e..7883c7b3dd 100644 --- a/docs/01_introduction/09_running_in_cloud.mdx +++ b/docs/02_introduction/09_running_in_cloud.mdx @@ -12,7 +12,7 @@ import MainExample from '!!raw-loader!./code_examples/09_apify_sdk.py'; Crawlee is developed by [**Apify**](https://apify.com), the web scraping and automation platform. You could say it is the **home of Crawlee projects**. In this section you'll see how to deploy the crawler there with just a few simple steps. You can deploy a **Crawlee** project wherever you want, but using the [**Apify platform**](https://console.apify.com) will give you the best experience. -To deploy your project to other platforms, see [Deploying to AWS Lambda](../guides/deploy-on-aws-lambda) and [Deploying to Google Cloud Run](../guides/deploy-on-gcp-cloud-run). +To deploy your project to other platforms, see the [Deployment](../deployment) section. With a few simple steps, you can convert your Crawlee project into a so-called **Actor**. Actors are serverless micro-apps that are easy to develop, run, share, and integrate. The infra, proxies, and storages are ready to go. [Learn more about Actors](https://apify.com/actors). diff --git a/docs/01_introduction/code_examples/02_bs.py b/docs/02_introduction/code_examples/02_bs.py similarity index 100% rename from docs/01_introduction/code_examples/02_bs.py rename to docs/02_introduction/code_examples/02_bs.py diff --git a/docs/01_introduction/code_examples/02_bs_better.py b/docs/02_introduction/code_examples/02_bs_better.py similarity index 100% rename from docs/01_introduction/code_examples/02_bs_better.py rename to docs/02_introduction/code_examples/02_bs_better.py diff --git a/docs/01_introduction/code_examples/02_request_queue.py b/docs/02_introduction/code_examples/02_request_queue.py similarity index 100% rename from docs/01_introduction/code_examples/02_request_queue.py rename to docs/02_introduction/code_examples/02_request_queue.py diff --git a/docs/01_introduction/code_examples/03_enqueue_strategy.py b/docs/02_introduction/code_examples/03_enqueue_strategy.py similarity index 100% rename from docs/01_introduction/code_examples/03_enqueue_strategy.py rename to docs/02_introduction/code_examples/03_enqueue_strategy.py diff --git a/docs/01_introduction/code_examples/03_finding_new_links.py b/docs/02_introduction/code_examples/03_finding_new_links.py similarity index 100% rename from docs/01_introduction/code_examples/03_finding_new_links.py rename to docs/02_introduction/code_examples/03_finding_new_links.py diff --git a/docs/01_introduction/code_examples/03_globs.py b/docs/02_introduction/code_examples/03_globs.py similarity index 100% rename from docs/01_introduction/code_examples/03_globs.py rename to docs/02_introduction/code_examples/03_globs.py diff --git a/docs/01_introduction/code_examples/03_original_code.py b/docs/02_introduction/code_examples/03_original_code.py similarity index 100% rename from docs/01_introduction/code_examples/03_original_code.py rename to docs/02_introduction/code_examples/03_original_code.py diff --git a/docs/01_introduction/code_examples/03_transform_request.py b/docs/02_introduction/code_examples/03_transform_request.py similarity index 100% rename from docs/01_introduction/code_examples/03_transform_request.py rename to docs/02_introduction/code_examples/03_transform_request.py diff --git a/docs/01_introduction/code_examples/04_sanity_check.py b/docs/02_introduction/code_examples/04_sanity_check.py similarity index 100% rename from docs/01_introduction/code_examples/04_sanity_check.py rename to docs/02_introduction/code_examples/04_sanity_check.py diff --git a/docs/01_introduction/code_examples/05_crawling_detail.py b/docs/02_introduction/code_examples/05_crawling_detail.py similarity index 100% rename from docs/01_introduction/code_examples/05_crawling_detail.py rename to docs/02_introduction/code_examples/05_crawling_detail.py diff --git a/docs/01_introduction/code_examples/05_crawling_listing.py b/docs/02_introduction/code_examples/05_crawling_listing.py similarity index 100% rename from docs/01_introduction/code_examples/05_crawling_listing.py rename to docs/02_introduction/code_examples/05_crawling_listing.py diff --git a/docs/01_introduction/code_examples/06_scraping.py b/docs/02_introduction/code_examples/06_scraping.py similarity index 100% rename from docs/01_introduction/code_examples/06_scraping.py rename to docs/02_introduction/code_examples/06_scraping.py diff --git a/docs/01_introduction/code_examples/07_final_code.py b/docs/02_introduction/code_examples/07_final_code.py similarity index 100% rename from docs/01_introduction/code_examples/07_final_code.py rename to docs/02_introduction/code_examples/07_final_code.py diff --git a/docs/01_introduction/code_examples/07_first_code.py b/docs/02_introduction/code_examples/07_first_code.py similarity index 100% rename from docs/01_introduction/code_examples/07_first_code.py rename to docs/02_introduction/code_examples/07_first_code.py diff --git a/docs/01_introduction/code_examples/08_main.py b/docs/02_introduction/code_examples/08_main.py similarity index 100% rename from docs/01_introduction/code_examples/08_main.py rename to docs/02_introduction/code_examples/08_main.py diff --git a/docs/01_introduction/code_examples/08_routes.py b/docs/02_introduction/code_examples/08_routes.py similarity index 100% rename from docs/01_introduction/code_examples/08_routes.py rename to docs/02_introduction/code_examples/08_routes.py diff --git a/docs/01_introduction/code_examples/09_apify_sdk.py b/docs/02_introduction/code_examples/09_apify_sdk.py similarity index 100% rename from docs/01_introduction/code_examples/09_apify_sdk.py rename to docs/02_introduction/code_examples/09_apify_sdk.py diff --git a/docs/01_introduction/code_examples/__init__.py b/docs/02_introduction/code_examples/__init__.py similarity index 100% rename from docs/01_introduction/code_examples/__init__.py rename to docs/02_introduction/code_examples/__init__.py diff --git a/docs/01_introduction/code_examples/routes.py b/docs/02_introduction/code_examples/routes.py similarity index 100% rename from docs/01_introduction/code_examples/routes.py rename to docs/02_introduction/code_examples/routes.py diff --git a/docs/02_introduction/index.mdx b/docs/02_introduction/index.mdx new file mode 100644 index 0000000000..fc7d3dab86 --- /dev/null +++ b/docs/02_introduction/index.mdx @@ -0,0 +1,56 @@ +--- +id: introduction +title: Introduction +description: A step-by-step tutorial that takes you from your first crawler to a production-ready scraper for a real website. +--- + +import ApiLink from '@site/src/components/ApiLink'; + +Crawlee covers your crawling and scraping end-to-end and helps you **build reliable scrapers. Fast.** + +Your crawlers will appear human-like and fly under the radar of modern bot protections even with the default configuration. Crawlee gives you the tools to crawl the web for links, scrape data and persistently store it in machine-readable formats, without having to worry about the technical details. And thanks to rich configuration options, you can tweak almost any aspect of Crawlee to suit your project's needs if the default settings don't cut it. + +## What you will learn + +The goal of the introduction is to provide a step-by-step guide to the most important features of Crawlee. It will walk you through creating the simplest of crawlers that only prints text to console, all the way up to a full-featured scraper that collects links from a website and extracts data. + +## ๐Ÿ›  Features + +Why Crawlee is the preferred choice for web scraping and crawling? + +### Why use Crawlee instead of just a random HTTP library with an HTML parser? + +- Unified interface for **HTTP & headless browser** crawling. +- Automatic **parallel crawling** based on available system resources. +- Written in Python with **type hints** - enhances DX (IDE autocompletion) and reduces bugs (static type checking). +- Automatic **retries** on errors or when you are getting blocked. +- Integrated **proxy rotation** and session management. +- Configurable **request routing** - direct URLs to the appropriate handlers. +- Persistent **queue for URLs** to crawl. +- Pluggable **storage** of both tabular data and files. +- Robust **error handling**. + +### Why to use Crawlee rather than Scrapy? + +- Crawlee has out-of-the-box support for **headless browser** crawling (Playwright). +- Crawlee has a **minimalistic & elegant interface** - Set up your scraper with fewer than 10 lines of code. +- Complete **type hint** coverage. +- Based on standard **Asyncio**. + +{/* TODO: + +### ๐Ÿ‘พ HTTP crawling + +- ... +*/} + +{/* TODO: + +### ๐Ÿ’ป Real browser crawling + +- ... +*/} + +## Next steps + +Next, you will install Crawlee and learn how to bootstrap projects with the prepared Crawlee templates. diff --git a/docs/02_concepts/01_architecture_overview.mdx b/docs/03_concepts/01_architecture_overview.mdx similarity index 100% rename from docs/02_concepts/01_architecture_overview.mdx rename to docs/03_concepts/01_architecture_overview.mdx diff --git a/docs/02_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx similarity index 100% rename from docs/02_concepts/02_requests.mdx rename to docs/03_concepts/02_requests.mdx diff --git a/docs/02_concepts/03_crawling_context.mdx b/docs/03_concepts/03_crawling_context.mdx similarity index 100% rename from docs/02_concepts/03_crawling_context.mdx rename to docs/03_concepts/03_crawling_context.mdx diff --git a/docs/02_concepts/04_request_router.mdx b/docs/03_concepts/04_request_router.mdx similarity index 100% rename from docs/02_concepts/04_request_router.mdx rename to docs/03_concepts/04_request_router.mdx diff --git a/docs/02_concepts/05_http_crawlers.mdx b/docs/03_concepts/05_http_crawlers.mdx similarity index 100% rename from docs/02_concepts/05_http_crawlers.mdx rename to docs/03_concepts/05_http_crawlers.mdx diff --git a/docs/02_concepts/06_playwright_crawler.mdx b/docs/03_concepts/06_playwright_crawler.mdx similarity index 100% rename from docs/02_concepts/06_playwright_crawler.mdx rename to docs/03_concepts/06_playwright_crawler.mdx diff --git a/docs/02_concepts/07_playwright_crawler_adaptive.mdx b/docs/03_concepts/07_playwright_crawler_adaptive.mdx similarity index 100% rename from docs/02_concepts/07_playwright_crawler_adaptive.mdx rename to docs/03_concepts/07_playwright_crawler_adaptive.mdx diff --git a/docs/02_concepts/08_pydantic_ai_crawler.mdx b/docs/03_concepts/08_pydantic_ai_crawler.mdx similarity index 100% rename from docs/02_concepts/08_pydantic_ai_crawler.mdx rename to docs/03_concepts/08_pydantic_ai_crawler.mdx diff --git a/docs/02_concepts/09_stagehand_crawler.mdx b/docs/03_concepts/09_stagehand_crawler.mdx similarity index 100% rename from docs/02_concepts/09_stagehand_crawler.mdx rename to docs/03_concepts/09_stagehand_crawler.mdx diff --git a/docs/02_concepts/10_request_loaders.mdx b/docs/03_concepts/10_request_loaders.mdx similarity index 100% rename from docs/02_concepts/10_request_loaders.mdx rename to docs/03_concepts/10_request_loaders.mdx diff --git a/docs/02_concepts/11_storages.mdx b/docs/03_concepts/11_storages.mdx similarity index 100% rename from docs/02_concepts/11_storages.mdx rename to docs/03_concepts/11_storages.mdx diff --git a/docs/02_concepts/12_storage_clients.mdx b/docs/03_concepts/12_storage_clients.mdx similarity index 100% rename from docs/02_concepts/12_storage_clients.mdx rename to docs/03_concepts/12_storage_clients.mdx diff --git a/docs/02_concepts/13_http_clients.mdx b/docs/03_concepts/13_http_clients.mdx similarity index 100% rename from docs/02_concepts/13_http_clients.mdx rename to docs/03_concepts/13_http_clients.mdx diff --git a/docs/02_concepts/14_session_management.mdx b/docs/03_concepts/14_session_management.mdx similarity index 100% rename from docs/02_concepts/14_session_management.mdx rename to docs/03_concepts/14_session_management.mdx diff --git a/docs/02_concepts/15_cookie_management.mdx b/docs/03_concepts/15_cookie_management.mdx similarity index 100% rename from docs/02_concepts/15_cookie_management.mdx rename to docs/03_concepts/15_cookie_management.mdx diff --git a/docs/02_concepts/16_http_headers.mdx b/docs/03_concepts/16_http_headers.mdx similarity index 100% rename from docs/02_concepts/16_http_headers.mdx rename to docs/03_concepts/16_http_headers.mdx diff --git a/docs/02_concepts/17_proxy_management.mdx b/docs/03_concepts/17_proxy_management.mdx similarity index 100% rename from docs/02_concepts/17_proxy_management.mdx rename to docs/03_concepts/17_proxy_management.mdx diff --git a/docs/02_concepts/18_scaling_crawlers.mdx b/docs/03_concepts/18_scaling_crawlers.mdx similarity index 100% rename from docs/02_concepts/18_scaling_crawlers.mdx rename to docs/03_concepts/18_scaling_crawlers.mdx diff --git a/docs/02_concepts/19_request_throttling.mdx b/docs/03_concepts/19_request_throttling.mdx similarity index 100% rename from docs/02_concepts/19_request_throttling.mdx rename to docs/03_concepts/19_request_throttling.mdx diff --git a/docs/02_concepts/20_error_handling.mdx b/docs/03_concepts/20_error_handling.mdx similarity index 100% rename from docs/02_concepts/20_error_handling.mdx rename to docs/03_concepts/20_error_handling.mdx diff --git a/docs/02_concepts/21_events.mdx b/docs/03_concepts/21_events.mdx similarity index 100% rename from docs/02_concepts/21_events.mdx rename to docs/03_concepts/21_events.mdx diff --git a/docs/02_concepts/22_statistics.mdx b/docs/03_concepts/22_statistics.mdx similarity index 100% rename from docs/02_concepts/22_statistics.mdx rename to docs/03_concepts/22_statistics.mdx diff --git a/docs/02_concepts/23_logging.mdx b/docs/03_concepts/23_logging.mdx similarity index 100% rename from docs/02_concepts/23_logging.mdx rename to docs/03_concepts/23_logging.mdx diff --git a/docs/02_concepts/24_configuration.mdx b/docs/03_concepts/24_configuration.mdx similarity index 100% rename from docs/02_concepts/24_configuration.mdx rename to docs/03_concepts/24_configuration.mdx diff --git a/docs/02_concepts/25_service_locator.mdx b/docs/03_concepts/25_service_locator.mdx similarity index 100% rename from docs/02_concepts/25_service_locator.mdx rename to docs/03_concepts/25_service_locator.mdx diff --git a/docs/02_concepts/26_extending_crawlee.mdx b/docs/03_concepts/26_extending_crawlee.mdx similarity index 100% rename from docs/02_concepts/26_extending_crawlee.mdx rename to docs/03_concepts/26_extending_crawlee.mdx diff --git a/docs/02_concepts/code_examples/configuration/crawler_configuration.py b/docs/03_concepts/code_examples/configuration/crawler_configuration.py similarity index 100% rename from docs/02_concepts/code_examples/configuration/crawler_configuration.py rename to docs/03_concepts/code_examples/configuration/crawler_configuration.py diff --git a/docs/02_concepts/code_examples/configuration/global_configuration.py b/docs/03_concepts/code_examples/configuration/global_configuration.py similarity index 100% rename from docs/02_concepts/code_examples/configuration/global_configuration.py rename to docs/03_concepts/code_examples/configuration/global_configuration.py diff --git a/docs/02_concepts/code_examples/cookie_management/initial_cookies.py b/docs/03_concepts/code_examples/cookie_management/initial_cookies.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/initial_cookies.py rename to docs/03_concepts/code_examples/cookie_management/initial_cookies.py diff --git a/docs/02_concepts/code_examples/cookie_management/persist_cookies.py b/docs/03_concepts/code_examples/cookie_management/persist_cookies.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/persist_cookies.py rename to docs/03_concepts/code_examples/cookie_management/persist_cookies.py diff --git a/docs/02_concepts/code_examples/cookie_management/playwright_cookies.py b/docs/03_concepts/code_examples/cookie_management/playwright_cookies.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/playwright_cookies.py rename to docs/03_concepts/code_examples/cookie_management/playwright_cookies.py diff --git a/docs/02_concepts/code_examples/cookie_management/read_write_cookies.py b/docs/03_concepts/code_examples/cookie_management/read_write_cookies.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/read_write_cookies.py rename to docs/03_concepts/code_examples/cookie_management/read_write_cookies.py diff --git a/docs/02_concepts/code_examples/cookie_management/retry_pinned_session.py b/docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/retry_pinned_session.py rename to docs/03_concepts/code_examples/cookie_management/retry_pinned_session.py diff --git a/docs/02_concepts/code_examples/cookie_management/retry_restore_cookies.py b/docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/retry_restore_cookies.py rename to docs/03_concepts/code_examples/cookie_management/retry_restore_cookies.py diff --git a/docs/02_concepts/code_examples/cookie_management/retry_single_session.py b/docs/03_concepts/code_examples/cookie_management/retry_single_session.py similarity index 100% rename from docs/02_concepts/code_examples/cookie_management/retry_single_session.py rename to docs/03_concepts/code_examples/cookie_management/retry_single_session.py diff --git a/docs/02_concepts/code_examples/crawling_context/buffered_results.py b/docs/03_concepts/code_examples/crawling_context/buffered_results.py similarity index 100% rename from docs/02_concepts/code_examples/crawling_context/buffered_results.py rename to docs/03_concepts/code_examples/crawling_context/buffered_results.py diff --git a/docs/02_concepts/code_examples/crawling_context/context_helpers.py b/docs/03_concepts/code_examples/crawling_context/context_helpers.py similarity index 100% rename from docs/02_concepts/code_examples/crawling_context/context_helpers.py rename to docs/03_concepts/code_examples/crawling_context/context_helpers.py diff --git a/docs/02_concepts/code_examples/error_handling/change_handle_error_status.py b/docs/03_concepts/code_examples/error_handling/change_handle_error_status.py similarity index 100% rename from docs/02_concepts/code_examples/error_handling/change_handle_error_status.py rename to docs/03_concepts/code_examples/error_handling/change_handle_error_status.py diff --git a/docs/02_concepts/code_examples/error_handling/disable_retry.py b/docs/03_concepts/code_examples/error_handling/disable_retry.py similarity index 100% rename from docs/02_concepts/code_examples/error_handling/disable_retry.py rename to docs/03_concepts/code_examples/error_handling/disable_retry.py diff --git a/docs/02_concepts/code_examples/error_handling/handle_proxy_error.py b/docs/03_concepts/code_examples/error_handling/handle_proxy_error.py similarity index 100% rename from docs/02_concepts/code_examples/error_handling/handle_proxy_error.py rename to docs/03_concepts/code_examples/error_handling/handle_proxy_error.py diff --git a/docs/02_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py b/docs/03_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py similarity index 100% rename from docs/02_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py rename to docs/03_concepts/code_examples/error_handling/parsel_crawler_with_error_snapshotter.py diff --git a/docs/02_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py b/docs/03_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py similarity index 100% rename from docs/02_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py rename to docs/03_concepts/code_examples/error_handling/playwright_crawler_with_error_snapshotter.py diff --git a/docs/02_concepts/code_examples/events/listen_to_events.py b/docs/03_concepts/code_examples/events/listen_to_events.py similarity index 100% rename from docs/02_concepts/code_examples/events/listen_to_events.py rename to docs/03_concepts/code_examples/events/listen_to_events.py diff --git a/docs/02_concepts/code_examples/events/use_state.py b/docs/03_concepts/code_examples/events/use_state.py similarity index 100% rename from docs/02_concepts/code_examples/events/use_state.py rename to docs/03_concepts/code_examples/events/use_state.py diff --git a/docs/02_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py b/docs/03_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py rename to docs/03_concepts/code_examples/http_clients/parsel_curl_impersonate_example.py diff --git a/docs/02_concepts/code_examples/http_clients/parsel_httpx_example.py b/docs/03_concepts/code_examples/http_clients/parsel_httpx_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_clients/parsel_httpx_example.py rename to docs/03_concepts/code_examples/http_clients/parsel_httpx_example.py diff --git a/docs/02_concepts/code_examples/http_clients/parsel_impit_example.py b/docs/03_concepts/code_examples/http_clients/parsel_impit_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_clients/parsel_impit_example.py rename to docs/03_concepts/code_examples/http_clients/parsel_impit_example.py diff --git a/docs/02_concepts/code_examples/http_crawlers/__init__.py b/docs/03_concepts/code_examples/http_crawlers/__init__.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/__init__.py rename to docs/03_concepts/code_examples/http_crawlers/__init__.py diff --git a/docs/02_concepts/code_examples/http_crawlers/beautifulsoup_example.py b/docs/03_concepts/code_examples/http_crawlers/beautifulsoup_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/beautifulsoup_example.py rename to docs/03_concepts/code_examples/http_crawlers/beautifulsoup_example.py diff --git a/docs/02_concepts/code_examples/http_crawlers/custom_crawler_example.py b/docs/03_concepts/code_examples/http_crawlers/custom_crawler_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/custom_crawler_example.py rename to docs/03_concepts/code_examples/http_crawlers/custom_crawler_example.py diff --git a/docs/02_concepts/code_examples/http_crawlers/file_download.py b/docs/03_concepts/code_examples/http_crawlers/file_download.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/file_download.py rename to docs/03_concepts/code_examples/http_crawlers/file_download.py diff --git a/docs/02_concepts/code_examples/http_crawlers/file_download_stream.py b/docs/03_concepts/code_examples/http_crawlers/file_download_stream.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/file_download_stream.py rename to docs/03_concepts/code_examples/http_crawlers/file_download_stream.py diff --git a/docs/02_concepts/code_examples/http_crawlers/http_example.py b/docs/03_concepts/code_examples/http_crawlers/http_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/http_example.py rename to docs/03_concepts/code_examples/http_crawlers/http_example.py diff --git a/docs/02_concepts/code_examples/http_crawlers/lexbor_parser.py b/docs/03_concepts/code_examples/http_crawlers/lexbor_parser.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/lexbor_parser.py rename to docs/03_concepts/code_examples/http_crawlers/lexbor_parser.py diff --git a/docs/02_concepts/code_examples/http_crawlers/lxml_parser.py b/docs/03_concepts/code_examples/http_crawlers/lxml_parser.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/lxml_parser.py rename to docs/03_concepts/code_examples/http_crawlers/lxml_parser.py diff --git a/docs/02_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py b/docs/03_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py rename to docs/03_concepts/code_examples/http_crawlers/lxml_saxonche_parser.py diff --git a/docs/02_concepts/code_examples/http_crawlers/parsel_example.py b/docs/03_concepts/code_examples/http_crawlers/parsel_example.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/parsel_example.py rename to docs/03_concepts/code_examples/http_crawlers/parsel_example.py diff --git a/docs/02_concepts/code_examples/http_crawlers/pyquery_parser.py b/docs/03_concepts/code_examples/http_crawlers/pyquery_parser.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/pyquery_parser.py rename to docs/03_concepts/code_examples/http_crawlers/pyquery_parser.py diff --git a/docs/02_concepts/code_examples/http_crawlers/scrapling_parser.py b/docs/03_concepts/code_examples/http_crawlers/scrapling_parser.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/scrapling_parser.py rename to docs/03_concepts/code_examples/http_crawlers/scrapling_parser.py diff --git a/docs/02_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_adaptive_run.py diff --git a/docs/02_concepts/code_examples/http_crawlers/selectolax_context.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_context.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/selectolax_context.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_context.py diff --git a/docs/02_concepts/code_examples/http_crawlers/selectolax_crawler.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_crawler.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/selectolax_crawler.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_crawler.py diff --git a/docs/02_concepts/code_examples/http_crawlers/selectolax_crawler_run.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_crawler_run.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/selectolax_crawler_run.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_crawler_run.py diff --git a/docs/02_concepts/code_examples/http_crawlers/selectolax_parser.py b/docs/03_concepts/code_examples/http_crawlers/selectolax_parser.py similarity index 100% rename from docs/02_concepts/code_examples/http_crawlers/selectolax_parser.py rename to docs/03_concepts/code_examples/http_crawlers/selectolax_parser.py diff --git a/docs/02_concepts/code_examples/http_headers/browser_page_headers.py b/docs/03_concepts/code_examples/http_headers/browser_page_headers.py similarity index 100% rename from docs/02_concepts/code_examples/http_headers/browser_page_headers.py rename to docs/03_concepts/code_examples/http_headers/browser_page_headers.py diff --git a/docs/02_concepts/code_examples/http_headers/set_headers.py b/docs/03_concepts/code_examples/http_headers/set_headers.py similarity index 100% rename from docs/02_concepts/code_examples/http_headers/set_headers.py rename to docs/03_concepts/code_examples/http_headers/set_headers.py diff --git a/docs/02_concepts/code_examples/logging/configure_json_logging.py b/docs/03_concepts/code_examples/logging/configure_json_logging.py similarity index 100% rename from docs/02_concepts/code_examples/logging/configure_json_logging.py rename to docs/03_concepts/code_examples/logging/configure_json_logging.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/browser_configuration_example.py b/docs/03_concepts/code_examples/playwright_crawler/browser_configuration_example.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/browser_configuration_example.py rename to docs/03_concepts/code_examples/playwright_crawler/browser_configuration_example.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py b/docs/03_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py rename to docs/03_concepts/code_examples/playwright_crawler/browser_pool_launch_hooks_example.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py b/docs/03_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py rename to docs/03_concepts/code_examples/playwright_crawler/browser_pool_page_hooks_example.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py b/docs/03_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py rename to docs/03_concepts/code_examples/playwright_crawler/capture_screenshot_using_playwright.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/multiple_launch_example.py b/docs/03_concepts/code_examples/playwright_crawler/multiple_launch_example.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/multiple_launch_example.py rename to docs/03_concepts/code_examples/playwright_crawler/multiple_launch_example.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/navigation_hooks_example.py b/docs/03_concepts/code_examples/playwright_crawler/navigation_hooks_example.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/navigation_hooks_example.py rename to docs/03_concepts/code_examples/playwright_crawler/navigation_hooks_example.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/playwright_block_requests.py b/docs/03_concepts/code_examples/playwright_crawler/playwright_block_requests.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/playwright_block_requests.py rename to docs/03_concepts/code_examples/playwright_crawler/playwright_block_requests.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/playwright_crawler.py b/docs/03_concepts/code_examples/playwright_crawler/playwright_crawler.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/playwright_crawler.py rename to docs/03_concepts/code_examples/playwright_crawler/playwright_crawler.py diff --git a/docs/02_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py b/docs/03_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py rename to docs/03_concepts/code_examples/playwright_crawler/plugin_browser_configuration_example.py diff --git a/docs/02_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/adaptive_playwright_crawler.py diff --git a/docs/02_concepts/code_examples/playwright_crawler_adaptive/handler.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/handler.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler_adaptive/handler.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/handler.py diff --git a/docs/02_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/init_beautifulsoup.py diff --git a/docs/02_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/init_parsel.py diff --git a/docs/02_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/init_prediction.py diff --git a/docs/02_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py b/docs/03_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py similarity index 100% rename from docs/02_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py rename to docs/03_concepts/code_examples/playwright_crawler_adaptive/pre_nav_hooks.py diff --git a/docs/02_concepts/code_examples/proxy_management/inspecting_bs_example.py b/docs/03_concepts/code_examples/proxy_management/inspecting_bs_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/inspecting_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/inspecting_bs_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/inspecting_pw_example.py b/docs/03_concepts/code_examples/proxy_management/inspecting_pw_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/inspecting_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/inspecting_pw_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/integration_bs_example.py b/docs/03_concepts/code_examples/proxy_management/integration_bs_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/integration_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/integration_bs_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/integration_pw_example.py b/docs/03_concepts/code_examples/proxy_management/integration_pw_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/integration_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/integration_pw_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/quick_start_example.py b/docs/03_concepts/code_examples/proxy_management/quick_start_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/quick_start_example.py rename to docs/03_concepts/code_examples/proxy_management/quick_start_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/session_bs_example.py b/docs/03_concepts/code_examples/proxy_management/session_bs_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/session_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/session_bs_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/session_pw_example.py b/docs/03_concepts/code_examples/proxy_management/session_pw_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/session_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/session_pw_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/tiers_bs_example.py b/docs/03_concepts/code_examples/proxy_management/tiers_bs_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/tiers_bs_example.py rename to docs/03_concepts/code_examples/proxy_management/tiers_bs_example.py diff --git a/docs/02_concepts/code_examples/proxy_management/tiers_pw_example.py b/docs/03_concepts/code_examples/proxy_management/tiers_pw_example.py similarity index 100% rename from docs/02_concepts/code_examples/proxy_management/tiers_pw_example.py rename to docs/03_concepts/code_examples/proxy_management/tiers_pw_example.py diff --git a/docs/02_concepts/code_examples/pydantic_ai_crawler/additional_instructions_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/additional_instructions_example.py similarity index 100% rename from docs/02_concepts/code_examples/pydantic_ai_crawler/additional_instructions_example.py rename to docs/03_concepts/code_examples/pydantic_ai_crawler/additional_instructions_example.py diff --git a/docs/02_concepts/code_examples/pydantic_ai_crawler/basic_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/pydantic_ai_crawler/basic_example.py rename to docs/03_concepts/code_examples/pydantic_ai_crawler/basic_example.py diff --git a/docs/02_concepts/code_examples/pydantic_ai_crawler/custom_distiller_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/custom_distiller_example.py similarity index 100% rename from docs/02_concepts/code_examples/pydantic_ai_crawler/custom_distiller_example.py rename to docs/03_concepts/code_examples/pydantic_ai_crawler/custom_distiller_example.py diff --git a/docs/02_concepts/code_examples/pydantic_ai_crawler/debugging_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/debugging_example.py similarity index 100% rename from docs/02_concepts/code_examples/pydantic_ai_crawler/debugging_example.py rename to docs/03_concepts/code_examples/pydantic_ai_crawler/debugging_example.py diff --git a/docs/02_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py similarity index 100% rename from docs/02_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py rename to docs/03_concepts/code_examples/pydantic_ai_crawler/selector_extractor_example.py diff --git a/docs/02_concepts/code_examples/pydantic_ai_crawler/usage_limit_example.py b/docs/03_concepts/code_examples/pydantic_ai_crawler/usage_limit_example.py similarity index 100% rename from docs/02_concepts/code_examples/pydantic_ai_crawler/usage_limit_example.py rename to docs/03_concepts/code_examples/pydantic_ai_crawler/usage_limit_example.py diff --git a/docs/02_concepts/code_examples/request_loaders/rl_basic_example.py b/docs/03_concepts/code_examples/request_loaders/rl_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/rl_basic_example.py rename to docs/03_concepts/code_examples/request_loaders/rl_basic_example.py diff --git a/docs/02_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py b/docs/03_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py rename to docs/03_concepts/code_examples/request_loaders/rl_basic_example_with_persist.py diff --git a/docs/02_concepts/code_examples/request_loaders/rl_tandem_example.py b/docs/03_concepts/code_examples/request_loaders/rl_tandem_example.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/rl_tandem_example.py rename to docs/03_concepts/code_examples/request_loaders/rl_tandem_example.py diff --git a/docs/02_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py b/docs/03_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py rename to docs/03_concepts/code_examples/request_loaders/rl_tandem_example_explicit.py diff --git a/docs/02_concepts/code_examples/request_loaders/sitemap_basic_example.py b/docs/03_concepts/code_examples/request_loaders/sitemap_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/sitemap_basic_example.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_basic_example.py diff --git a/docs/02_concepts/code_examples/request_loaders/sitemap_example_with_persist.py b/docs/03_concepts/code_examples/request_loaders/sitemap_example_with_persist.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/sitemap_example_with_persist.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_example_with_persist.py diff --git a/docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example.py b/docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example.py diff --git a/docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py b/docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py rename to docs/03_concepts/code_examples/request_loaders/sitemap_tandem_example_explicit.py diff --git a/docs/02_concepts/code_examples/request_loaders/using_sitemap_request_loader.py b/docs/03_concepts/code_examples/request_loaders/using_sitemap_request_loader.py similarity index 100% rename from docs/02_concepts/code_examples/request_loaders/using_sitemap_request_loader.py rename to docs/03_concepts/code_examples/request_loaders/using_sitemap_request_loader.py diff --git a/docs/02_concepts/code_examples/request_router/adaptive_crawler_handlers.py b/docs/03_concepts/code_examples/request_router/adaptive_crawler_handlers.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/adaptive_crawler_handlers.py rename to docs/03_concepts/code_examples/request_router/adaptive_crawler_handlers.py diff --git a/docs/02_concepts/code_examples/request_router/basic_request_handlers.py b/docs/03_concepts/code_examples/request_router/basic_request_handlers.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/basic_request_handlers.py rename to docs/03_concepts/code_examples/request_router/basic_request_handlers.py diff --git a/docs/02_concepts/code_examples/request_router/custom_router_default_only.py b/docs/03_concepts/code_examples/request_router/custom_router_default_only.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/custom_router_default_only.py rename to docs/03_concepts/code_examples/request_router/custom_router_default_only.py diff --git a/docs/02_concepts/code_examples/request_router/error_handler.py b/docs/03_concepts/code_examples/request_router/error_handler.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/error_handler.py rename to docs/03_concepts/code_examples/request_router/error_handler.py diff --git a/docs/02_concepts/code_examples/request_router/failed_request_handler.py b/docs/03_concepts/code_examples/request_router/failed_request_handler.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/failed_request_handler.py rename to docs/03_concepts/code_examples/request_router/failed_request_handler.py diff --git a/docs/02_concepts/code_examples/request_router/http_pre_navigation.py b/docs/03_concepts/code_examples/request_router/http_pre_navigation.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/http_pre_navigation.py rename to docs/03_concepts/code_examples/request_router/http_pre_navigation.py diff --git a/docs/02_concepts/code_examples/request_router/playwright_pre_navigation.py b/docs/03_concepts/code_examples/request_router/playwright_pre_navigation.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/playwright_pre_navigation.py rename to docs/03_concepts/code_examples/request_router/playwright_pre_navigation.py diff --git a/docs/02_concepts/code_examples/request_router/router_middleware.py b/docs/03_concepts/code_examples/request_router/router_middleware.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/router_middleware.py rename to docs/03_concepts/code_examples/request_router/router_middleware.py diff --git a/docs/02_concepts/code_examples/request_router/simple_default_handler.py b/docs/03_concepts/code_examples/request_router/simple_default_handler.py similarity index 100% rename from docs/02_concepts/code_examples/request_router/simple_default_handler.py rename to docs/03_concepts/code_examples/request_router/simple_default_handler.py diff --git a/docs/02_concepts/code_examples/request_throttling/throttling_example.py b/docs/03_concepts/code_examples/request_throttling/throttling_example.py similarity index 100% rename from docs/02_concepts/code_examples/request_throttling/throttling_example.py rename to docs/03_concepts/code_examples/request_throttling/throttling_example.py diff --git a/docs/02_concepts/code_examples/requests/label_and_user_data.py b/docs/03_concepts/code_examples/requests/label_and_user_data.py similarity index 100% rename from docs/02_concepts/code_examples/requests/label_and_user_data.py rename to docs/03_concepts/code_examples/requests/label_and_user_data.py diff --git a/docs/02_concepts/code_examples/requests/unique_key_deduplication.py b/docs/03_concepts/code_examples/requests/unique_key_deduplication.py similarity index 100% rename from docs/02_concepts/code_examples/requests/unique_key_deduplication.py rename to docs/03_concepts/code_examples/requests/unique_key_deduplication.py diff --git a/docs/02_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py b/docs/03_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py similarity index 100% rename from docs/02_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py rename to docs/03_concepts/code_examples/scaling_crawlers/max_tasks_per_minute_example.py diff --git a/docs/02_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py b/docs/03_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py similarity index 100% rename from docs/02_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py rename to docs/03_concepts/code_examples/scaling_crawlers/min_and_max_concurrency_example.py diff --git a/docs/02_concepts/code_examples/service_locator/service_conflicts.py b/docs/03_concepts/code_examples/service_locator/service_conflicts.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_conflicts.py rename to docs/03_concepts/code_examples/service_locator/service_conflicts.py diff --git a/docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py b/docs/03_concepts/code_examples/service_locator/service_crawler_configuration.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_crawler_configuration.py rename to docs/03_concepts/code_examples/service_locator/service_crawler_configuration.py diff --git a/docs/02_concepts/code_examples/service_locator/service_crawler_event_manager.py b/docs/03_concepts/code_examples/service_locator/service_crawler_event_manager.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_crawler_event_manager.py rename to docs/03_concepts/code_examples/service_locator/service_crawler_event_manager.py diff --git a/docs/02_concepts/code_examples/service_locator/service_crawler_storage_client.py b/docs/03_concepts/code_examples/service_locator/service_crawler_storage_client.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_crawler_storage_client.py rename to docs/03_concepts/code_examples/service_locator/service_crawler_storage_client.py diff --git a/docs/02_concepts/code_examples/service_locator/service_locator_configuration.py b/docs/03_concepts/code_examples/service_locator/service_locator_configuration.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_locator_configuration.py rename to docs/03_concepts/code_examples/service_locator/service_locator_configuration.py diff --git a/docs/02_concepts/code_examples/service_locator/service_locator_event_manager.py b/docs/03_concepts/code_examples/service_locator/service_locator_event_manager.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_locator_event_manager.py rename to docs/03_concepts/code_examples/service_locator/service_locator_event_manager.py diff --git a/docs/02_concepts/code_examples/service_locator/service_locator_storage_client.py b/docs/03_concepts/code_examples/service_locator/service_locator_storage_client.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_locator_storage_client.py rename to docs/03_concepts/code_examples/service_locator/service_locator_storage_client.py diff --git a/docs/02_concepts/code_examples/service_locator/service_storage_configuration.py b/docs/03_concepts/code_examples/service_locator/service_storage_configuration.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_storage_configuration.py rename to docs/03_concepts/code_examples/service_locator/service_storage_configuration.py diff --git a/docs/02_concepts/code_examples/service_locator/service_storage_storage_client.py b/docs/03_concepts/code_examples/service_locator/service_storage_storage_client.py similarity index 100% rename from docs/02_concepts/code_examples/service_locator/service_storage_storage_client.py rename to docs/03_concepts/code_examples/service_locator/service_storage_storage_client.py diff --git a/docs/02_concepts/code_examples/session_management/multi_sessions_http.py b/docs/03_concepts/code_examples/session_management/multi_sessions_http.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/multi_sessions_http.py rename to docs/03_concepts/code_examples/session_management/multi_sessions_http.py diff --git a/docs/02_concepts/code_examples/session_management/one_session_http.py b/docs/03_concepts/code_examples/session_management/one_session_http.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/one_session_http.py rename to docs/03_concepts/code_examples/session_management/one_session_http.py diff --git a/docs/02_concepts/code_examples/session_management/sm_basic.py b/docs/03_concepts/code_examples/session_management/sm_basic.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/sm_basic.py rename to docs/03_concepts/code_examples/session_management/sm_basic.py diff --git a/docs/02_concepts/code_examples/session_management/sm_beautifulsoup.py b/docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/sm_beautifulsoup.py rename to docs/03_concepts/code_examples/session_management/sm_beautifulsoup.py diff --git a/docs/02_concepts/code_examples/session_management/sm_http.py b/docs/03_concepts/code_examples/session_management/sm_http.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/sm_http.py rename to docs/03_concepts/code_examples/session_management/sm_http.py diff --git a/docs/02_concepts/code_examples/session_management/sm_parsel.py b/docs/03_concepts/code_examples/session_management/sm_parsel.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/sm_parsel.py rename to docs/03_concepts/code_examples/session_management/sm_parsel.py diff --git a/docs/02_concepts/code_examples/session_management/sm_playwright.py b/docs/03_concepts/code_examples/session_management/sm_playwright.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/sm_playwright.py rename to docs/03_concepts/code_examples/session_management/sm_playwright.py diff --git a/docs/02_concepts/code_examples/session_management/sm_standalone.py b/docs/03_concepts/code_examples/session_management/sm_standalone.py similarity index 100% rename from docs/02_concepts/code_examples/session_management/sm_standalone.py rename to docs/03_concepts/code_examples/session_management/sm_standalone.py diff --git a/docs/02_concepts/code_examples/stagehand_crawler/basic_example.py b/docs/03_concepts/code_examples/stagehand_crawler/basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/stagehand_crawler/basic_example.py rename to docs/03_concepts/code_examples/stagehand_crawler/basic_example.py diff --git a/docs/02_concepts/code_examples/stagehand_crawler/browserbase_example.py b/docs/03_concepts/code_examples/stagehand_crawler/browserbase_example.py similarity index 100% rename from docs/02_concepts/code_examples/stagehand_crawler/browserbase_example.py rename to docs/03_concepts/code_examples/stagehand_crawler/browserbase_example.py diff --git a/docs/02_concepts/code_examples/statistics/custom_statistics.py b/docs/03_concepts/code_examples/statistics/custom_statistics.py similarity index 100% rename from docs/02_concepts/code_examples/statistics/custom_statistics.py rename to docs/03_concepts/code_examples/statistics/custom_statistics.py diff --git a/docs/02_concepts/code_examples/statistics/read_final_statistics.py b/docs/03_concepts/code_examples/statistics/read_final_statistics.py similarity index 100% rename from docs/02_concepts/code_examples/statistics/read_final_statistics.py rename to docs/03_concepts/code_examples/statistics/read_final_statistics.py diff --git a/docs/02_concepts/code_examples/storage_clients/custom_storage_client_example.py b/docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/custom_storage_client_example.py rename to docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/file_system_storage_client_basic_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py b/docs/03_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py rename to docs/03_concepts/code_examples/storage_clients/file_system_storage_client_configuration_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/memory_storage_client_basic_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/redis_storage_client_basic_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py b/docs/03_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py rename to docs/03_concepts/code_examples/storage_clients/redis_storage_client_configuration_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/registering_storage_clients_example.py b/docs/03_concepts/code_examples/storage_clients/registering_storage_clients_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/registering_storage_clients_example.py rename to docs/03_concepts/code_examples/storage_clients/registering_storage_clients_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py b/docs/03_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py rename to docs/03_concepts/code_examples/storage_clients/sql_storage_client_basic_example.py diff --git a/docs/02_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py b/docs/03_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py similarity index 100% rename from docs/02_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py rename to docs/03_concepts/code_examples/storage_clients/sql_storage_client_configuration_example.py diff --git a/docs/02_concepts/code_examples/storages/cleaning_do_not_purge_example.py b/docs/03_concepts/code_examples/storages/cleaning_do_not_purge_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/cleaning_do_not_purge_example.py rename to docs/03_concepts/code_examples/storages/cleaning_do_not_purge_example.py diff --git a/docs/02_concepts/code_examples/storages/cleaning_purge_explicitly_example.py b/docs/03_concepts/code_examples/storages/cleaning_purge_explicitly_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/cleaning_purge_explicitly_example.py rename to docs/03_concepts/code_examples/storages/cleaning_purge_explicitly_example.py diff --git a/docs/02_concepts/code_examples/storages/dataset_basic_example.py b/docs/03_concepts/code_examples/storages/dataset_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/dataset_basic_example.py rename to docs/03_concepts/code_examples/storages/dataset_basic_example.py diff --git a/docs/02_concepts/code_examples/storages/dataset_with_crawler_example.py b/docs/03_concepts/code_examples/storages/dataset_with_crawler_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/dataset_with_crawler_example.py rename to docs/03_concepts/code_examples/storages/dataset_with_crawler_example.py diff --git a/docs/02_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py b/docs/03_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py rename to docs/03_concepts/code_examples/storages/dataset_with_crawler_explicit_example.py diff --git a/docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py b/docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py similarity index 100% rename from docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py rename to docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_csv.py diff --git a/docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_json.py b/docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_json.py similarity index 100% rename from docs/02_concepts/code_examples/storages/export_entire_dataset_to_file_json.py rename to docs/03_concepts/code_examples/storages/export_entire_dataset_to_file_json.py diff --git a/docs/02_concepts/code_examples/storages/helper_add_requests_example.py b/docs/03_concepts/code_examples/storages/helper_add_requests_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/helper_add_requests_example.py rename to docs/03_concepts/code_examples/storages/helper_add_requests_example.py diff --git a/docs/02_concepts/code_examples/storages/helper_enqueue_links_example.py b/docs/03_concepts/code_examples/storages/helper_enqueue_links_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/helper_enqueue_links_example.py rename to docs/03_concepts/code_examples/storages/helper_enqueue_links_example.py diff --git a/docs/02_concepts/code_examples/storages/kvs_basic_example.py b/docs/03_concepts/code_examples/storages/kvs_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/kvs_basic_example.py rename to docs/03_concepts/code_examples/storages/kvs_basic_example.py diff --git a/docs/02_concepts/code_examples/storages/kvs_with_crawler_example.py b/docs/03_concepts/code_examples/storages/kvs_with_crawler_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/kvs_with_crawler_example.py rename to docs/03_concepts/code_examples/storages/kvs_with_crawler_example.py diff --git a/docs/02_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py b/docs/03_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py rename to docs/03_concepts/code_examples/storages/kvs_with_crawler_explicit_example.py diff --git a/docs/02_concepts/code_examples/storages/opening.py b/docs/03_concepts/code_examples/storages/opening.py similarity index 100% rename from docs/02_concepts/code_examples/storages/opening.py rename to docs/03_concepts/code_examples/storages/opening.py diff --git a/docs/02_concepts/code_examples/storages/rq_basic_example.py b/docs/03_concepts/code_examples/storages/rq_basic_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/rq_basic_example.py rename to docs/03_concepts/code_examples/storages/rq_basic_example.py diff --git a/docs/02_concepts/code_examples/storages/rq_with_crawler_example.py b/docs/03_concepts/code_examples/storages/rq_with_crawler_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/rq_with_crawler_example.py rename to docs/03_concepts/code_examples/storages/rq_with_crawler_example.py diff --git a/docs/02_concepts/code_examples/storages/rq_with_crawler_explicit_example.py b/docs/03_concepts/code_examples/storages/rq_with_crawler_explicit_example.py similarity index 100% rename from docs/02_concepts/code_examples/storages/rq_with_crawler_explicit_example.py rename to docs/03_concepts/code_examples/storages/rq_with_crawler_explicit_example.py diff --git a/docs/03_guides/01_crawling_links.mdx b/docs/04_guides/01_crawling_links.mdx similarity index 100% rename from docs/03_guides/01_crawling_links.mdx rename to docs/04_guides/01_crawling_links.mdx diff --git a/docs/03_guides/02_fill_and_submit_web_form.mdx b/docs/04_guides/02_fill_and_submit_web_form.mdx similarity index 100% rename from docs/03_guides/02_fill_and_submit_web_form.mdx rename to docs/04_guides/02_fill_and_submit_web_form.mdx diff --git a/docs/03_guides/03_crawler_login.mdx b/docs/04_guides/03_crawler_login.mdx similarity index 100% rename from docs/03_guides/03_crawler_login.mdx rename to docs/04_guides/03_crawler_login.mdx diff --git a/docs/03_guides/04_avoid_blocking.mdx b/docs/04_guides/04_avoid_blocking.mdx similarity index 100% rename from docs/03_guides/04_avoid_blocking.mdx rename to docs/04_guides/04_avoid_blocking.mdx diff --git a/docs/03_guides/05_respect_robots_txt_file.mdx b/docs/04_guides/05_respect_robots_txt_file.mdx similarity index 100% rename from docs/03_guides/05_respect_robots_txt_file.mdx rename to docs/04_guides/05_respect_robots_txt_file.mdx diff --git a/docs/03_guides/06_stopping_and_resuming.mdx b/docs/04_guides/06_stopping_and_resuming.mdx similarity index 100% rename from docs/03_guides/06_stopping_and_resuming.mdx rename to docs/04_guides/06_stopping_and_resuming.mdx diff --git a/docs/03_guides/07_run_parallel_crawlers.mdx b/docs/04_guides/07_run_parallel_crawlers.mdx similarity index 100% rename from docs/03_guides/07_run_parallel_crawlers.mdx rename to docs/04_guides/07_run_parallel_crawlers.mdx diff --git a/docs/03_guides/08_running_in_web_server.mdx b/docs/04_guides/08_running_in_web_server.mdx similarity index 100% rename from docs/03_guides/08_running_in_web_server.mdx rename to docs/04_guides/08_running_in_web_server.mdx diff --git a/docs/03_guides/09_trace_and_monitor_crawlers.mdx b/docs/04_guides/09_trace_and_monitor_crawlers.mdx similarity index 100% rename from docs/03_guides/09_trace_and_monitor_crawlers.mdx rename to docs/04_guides/09_trace_and_monitor_crawlers.mdx diff --git a/docs/03_guides/10_creating_web_archive.mdx b/docs/04_guides/10_creating_web_archive.mdx similarity index 100% rename from docs/03_guides/10_creating_web_archive.mdx rename to docs/04_guides/10_creating_web_archive.mdx diff --git a/docs/03_guides/11_secure_scraping.mdx b/docs/04_guides/11_secure_scraping.mdx similarity index 99% rename from docs/03_guides/11_secure_scraping.mdx rename to docs/04_guides/11_secure_scraping.mdx index 599e649f54..97b9b07279 100644 --- a/docs/03_guides/11_secure_scraping.mdx +++ b/docs/04_guides/11_secure_scraping.mdx @@ -114,7 +114,7 @@ Isolation also changes the calculus for the application-level controls. Inside a ### Running on the Apify platform -Building and maintaining isolated, egress-controlled runtimes is ongoing work. The [Apify platform](https://apify.com) runs each Actor (your crawler) in its own isolated container, without access to other users' data or to your internal network. That gives you container isolation and controlled egress out of the box. See the [Deploy on Apify](./deploy-on-apify) guide to run your Crawlee crawler there. +Building and maintaining isolated, egress-controlled runtimes is ongoing work. The [Apify platform](https://apify.com) runs each Actor (your crawler) in its own isolated container, without access to other users' data or to your internal network. That gives you container isolation and controlled egress out of the box. See the [Deploy on Apify](../deployment/apify-platform) guide to run your Crawlee crawler there. ## Conclusion diff --git a/docs/03_guides/12_scrapy_migration.mdx b/docs/04_guides/12_scrapy_migration.mdx similarity index 100% rename from docs/03_guides/12_scrapy_migration.mdx rename to docs/04_guides/12_scrapy_migration.mdx diff --git a/docs/03_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py b/docs/04_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py similarity index 100% rename from docs/03_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py rename to docs/04_guides/code_examples/avoid_blocking/default_fingerprint_generator_with_args.py diff --git a/docs/03_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py b/docs/04_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py similarity index 100% rename from docs/03_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py rename to docs/04_guides/code_examples/avoid_blocking/playwright_crawler_with_camoufox.py diff --git a/docs/03_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py b/docs/04_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py similarity index 100% rename from docs/03_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py rename to docs/04_guides/code_examples/avoid_blocking/playwright_with_cloakbrowser.py diff --git a/docs/03_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py b/docs/04_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py similarity index 100% rename from docs/03_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py rename to docs/04_guides/code_examples/avoid_blocking/playwright_with_fingerprint_generator.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py b/docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py rename to docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_bs.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py b/docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py rename to docs/04_guides/code_examples/crawling_links/crawl_all_links_on_website_pw.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py b/docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py rename to docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_bs.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py b/docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py rename to docs/04_guides/code_examples/crawling_links/crawl_multiple_urls_pw.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py b/docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py rename to docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_bs.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py b/docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py rename to docs/04_guides/code_examples/crawling_links/crawl_specific_links_on_website_pw.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_all_links.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_domain.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_hostname.py diff --git a/docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py b/docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py rename to docs/04_guides/code_examples/crawling_links/crawl_website_with_relative_links_same_origin.py diff --git a/docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py b/docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py rename to docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_bs.py diff --git a/docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py b/docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py similarity index 100% rename from docs/03_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py rename to docs/04_guides/code_examples/crawling_links/extract_and_add_specific_links_on_website_pw.py diff --git a/docs/03_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py b/docs/04_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py similarity index 100% rename from docs/03_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py rename to docs/04_guides/code_examples/creating_web_archive/manual_archiving_parsel_crawler.py diff --git a/docs/03_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py b/docs/04_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py similarity index 100% rename from docs/03_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py rename to docs/04_guides/code_examples/creating_web_archive/manual_archiving_playwright_crawler.py diff --git a/docs/03_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py b/docs/04_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py similarity index 100% rename from docs/03_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py rename to docs/04_guides/code_examples/creating_web_archive/simple_pw_through_proxy_pywb_server.py diff --git a/docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py b/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py similarity index 100% rename from docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py rename to docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py diff --git a/docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py b/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py similarity index 100% rename from docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py rename to docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py diff --git a/docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py b/docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py similarity index 100% rename from docs/03_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py rename to docs/04_guides/code_examples/fill_and_submit_web_form/fill_and_submit_web_form_request.py diff --git a/docs/03_guides/code_examples/login_crawler/http_login.py b/docs/04_guides/code_examples/login_crawler/http_login.py similarity index 100% rename from docs/03_guides/code_examples/login_crawler/http_login.py rename to docs/04_guides/code_examples/login_crawler/http_login.py diff --git a/docs/03_guides/code_examples/login_crawler/playwright_login.py b/docs/04_guides/code_examples/login_crawler/playwright_login.py similarity index 100% rename from docs/03_guides/code_examples/login_crawler/playwright_login.py rename to docs/04_guides/code_examples/login_crawler/playwright_login.py diff --git a/docs/03_guides/code_examples/login_crawler/using_browser_profiles_chrome.py b/docs/04_guides/code_examples/login_crawler/using_browser_profiles_chrome.py similarity index 100% rename from docs/03_guides/code_examples/login_crawler/using_browser_profiles_chrome.py rename to docs/04_guides/code_examples/login_crawler/using_browser_profiles_chrome.py diff --git a/docs/03_guides/code_examples/login_crawler/using_browser_profiles_firefox.py b/docs/04_guides/code_examples/login_crawler/using_browser_profiles_firefox.py similarity index 100% rename from docs/03_guides/code_examples/login_crawler/using_browser_profiles_firefox.py rename to docs/04_guides/code_examples/login_crawler/using_browser_profiles_firefox.py diff --git a/docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py b/docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py similarity index 100% rename from docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py rename to docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py diff --git a/docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py b/docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py similarity index 100% rename from docs/03_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py rename to docs/04_guides/code_examples/respect_robots_txt_file/respect_robots_txt_file.py diff --git a/docs/03_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py b/docs/04_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py similarity index 100% rename from docs/03_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py rename to docs/04_guides/code_examples/run_parallel_crawlers/run_parallel_crawlers.py diff --git a/docs/03_guides/code_examples/running_in_web_server/__init__.py b/docs/04_guides/code_examples/running_in_web_server/__init__.py similarity index 100% rename from docs/03_guides/code_examples/running_in_web_server/__init__.py rename to docs/04_guides/code_examples/running_in_web_server/__init__.py diff --git a/docs/03_guides/code_examples/running_in_web_server/crawler.py b/docs/04_guides/code_examples/running_in_web_server/crawler.py similarity index 100% rename from docs/03_guides/code_examples/running_in_web_server/crawler.py rename to docs/04_guides/code_examples/running_in_web_server/crawler.py diff --git a/docs/03_guides/code_examples/running_in_web_server/server.py b/docs/04_guides/code_examples/running_in_web_server/server.py similarity index 100% rename from docs/03_guides/code_examples/running_in_web_server/server.py rename to docs/04_guides/code_examples/running_in_web_server/server.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_concurrency.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_concurrency.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_concurrency.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_concurrency.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_crawlspider.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_crawlspider.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_crawlspider.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_crawlspider.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_error_handling.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_error_handling.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_error_handling.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_error_handling.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_export.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_export.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_export.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_export.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_labels.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_labels.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_labels.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_labels.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_playwright.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_playwright.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_playwright.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_playwright.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_post.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_post.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_post.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_post.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_proxy.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_proxy.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_proxy.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_proxy.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_quotes.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_quotes.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_quotes.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_quotes.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_throttling.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_throttling.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_throttling.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_throttling.py diff --git a/docs/03_guides/code_examples/scrapy_migration/crawlee_user_data.py b/docs/04_guides/code_examples/scrapy_migration/crawlee_user_data.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/crawlee_user_data.py rename to docs/04_guides/code_examples/scrapy_migration/crawlee_user_data.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_authors.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_authors.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_authors.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_authors.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_cb_kwargs.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_concurrency.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_concurrency.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_concurrency.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_concurrency.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_crawlspider.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_crawlspider.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_crawlspider.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_crawlspider.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_errback.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_errback.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_errback.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_errback.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_export.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_export.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_export.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_export.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_formrequest.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_formrequest.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_formrequest.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_formrequest.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_playwright.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_playwright.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_playwright.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_playwright.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_playwright_settings.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_proxy.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_proxy.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_proxy.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_proxy.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_quotes.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_quotes.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_quotes.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_quotes.py diff --git a/docs/03_guides/code_examples/scrapy_migration/scrapy_throttling.py b/docs/04_guides/code_examples/scrapy_migration/scrapy_throttling.py similarity index 100% rename from docs/03_guides/code_examples/scrapy_migration/scrapy_throttling.py rename to docs/04_guides/code_examples/scrapy_migration/scrapy_throttling.py diff --git a/docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py b/docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py similarity index 100% rename from docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py rename to docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_keep_alive.py diff --git a/docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py b/docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py similarity index 100% rename from docs/03_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py rename to docs/04_guides/code_examples/stopping_and_resuming/beautifulsoup_crawler_stop.py diff --git a/docs/03_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py b/docs/04_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py similarity index 100% rename from docs/03_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py rename to docs/04_guides/code_examples/stopping_and_resuming/resuming_paused_crawl.py diff --git a/docs/03_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py b/docs/04_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py similarity index 100% rename from docs/03_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py rename to docs/04_guides/code_examples/trace_and_monitor_crawlers/instrument_crawler.py diff --git a/docs/03_guides/13_deploy_on_apify.mdx b/docs/05_deployment/apify_platform.mdx similarity index 95% rename from docs/03_guides/13_deploy_on_apify.mdx rename to docs/05_deployment/apify_platform.mdx index 6d4f952e77..88db05ca7d 100644 --- a/docs/03_guides/13_deploy_on_apify.mdx +++ b/docs/05_deployment/apify_platform.mdx @@ -1,17 +1,17 @@ --- -id: deploy-on-apify -title: Deploying to Apify -description: How to run your Crawlee project as an Actor on the Apify platform, with cloud storages, Apify Proxy, and platform events. +id: apify-platform +title: Apify platform +description: Apify platform - large-scale and high-performance web scraping --- import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import LogWithConfigExample from '!!raw-loader!./code_examples/deployment/apify/log_with_config_example.py'; -import CrawlerAsActorExample from '!!raw-loader!./code_examples/deployment/apify/crawler_as_actor_example.py'; -import ProxyExample from '!!raw-loader!./code_examples/deployment/apify/proxy_example.py'; -import ProxyAdvancedExample from '!!raw-loader!./code_examples/deployment/apify/proxy_advanced_example.py'; +import LogWithConfigExample from '!!raw-loader!./code_examples/apify/log_with_config_example.py'; +import CrawlerAsActorExample from '!!raw-loader!./code_examples/apify/crawler_as_actor_example.py'; +import ProxyExample from '!!raw-loader!./code_examples/apify/proxy_example.py'; +import ProxyAdvancedExample from '!!raw-loader!./code_examples/apify/proxy_advanced_example.py'; Apify is a [platform](https://apify.com) built to serve large-scale and high-performance web scraping and automation needs. It provides easy access to [compute instances (Actors)](#what-is-an-actor), convenient request and result storages, [proxies](../concepts/proxy-management), scheduling, webhooks, and [more in the Apify documentation](https://docs.apify.com/), accessible through a [web interface](https://console.apify.com) or an [API](https://docs.apify.com/api). @@ -127,7 +127,7 @@ Your script will be uploaded to and built on the Apify platform so that it can b ## Usage on Apify platform -You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Guides](./.) section. +You can also develop your Actor in an online code editor directly on the platform (you'll need an Apify Account). Let's go to the [Actors](https://console.apify.com/actors) page in the app, click *Create new* and then go to the *Source* tab and start writing the code or paste one of the examples from the [Guides](../guides) section. ## Storages diff --git a/docs/03_guides/14_deploy_on_aws_lambda.mdx b/docs/05_deployment/aws_lambda.mdx similarity index 97% rename from docs/03_guides/14_deploy_on_aws_lambda.mdx rename to docs/05_deployment/aws_lambda.mdx index 9b71c73c4a..8d7d94123a 100644 --- a/docs/03_guides/14_deploy_on_aws_lambda.mdx +++ b/docs/05_deployment/aws_lambda.mdx @@ -1,6 +1,6 @@ --- -id: deploy-on-aws-lambda -title: Deploying to AWS Lambda +id: aws-lambda +title: Deploy on AWS Lambda description: Prepare your crawler to run on AWS Lambda. --- @@ -8,9 +8,9 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import BeautifulSoupCrawlerLambda from '!!raw-loader!./code_examples/deployment/aws/beautifulsoup_crawler_lambda.py'; -import PlaywrightCrawlerLambda from '!!raw-loader!./code_examples/deployment/aws/playwright_crawler_lambda.py'; -import PlaywrightCrawlerDockerfile from '!!raw-loader!./code_examples/deployment/aws/playwright_dockerfile'; +import BeautifulSoupCrawlerLambda from '!!raw-loader!./code_examples/aws/beautifulsoup_crawler_lambda.py'; +import PlaywrightCrawlerLambda from '!!raw-loader!./code_examples/aws/playwright_crawler_lambda.py'; +import PlaywrightCrawlerDockerfile from '!!raw-loader!./code_examples/aws/playwright_dockerfile'; [AWS Lambda](https://docs.aws.amazon.com/lambda/latest/dg/welcome.html) is a serverless compute service that lets you run code without provisioning or managing servers. This guide covers deploying `BeautifulSoupCrawler` and `PlaywrightCrawler`. diff --git a/docs/03_guides/code_examples/deployment/apify/crawler_as_actor_example.py b/docs/05_deployment/code_examples/apify/crawler_as_actor_example.py similarity index 100% rename from docs/03_guides/code_examples/deployment/apify/crawler_as_actor_example.py rename to docs/05_deployment/code_examples/apify/crawler_as_actor_example.py diff --git a/docs/03_guides/code_examples/deployment/apify/get_public_url.py b/docs/05_deployment/code_examples/apify/get_public_url.py similarity index 100% rename from docs/03_guides/code_examples/deployment/apify/get_public_url.py rename to docs/05_deployment/code_examples/apify/get_public_url.py diff --git a/docs/03_guides/code_examples/deployment/apify/log_with_config_example.py b/docs/05_deployment/code_examples/apify/log_with_config_example.py similarity index 100% rename from docs/03_guides/code_examples/deployment/apify/log_with_config_example.py rename to docs/05_deployment/code_examples/apify/log_with_config_example.py diff --git a/docs/03_guides/code_examples/deployment/apify/proxy_advanced_example.py b/docs/05_deployment/code_examples/apify/proxy_advanced_example.py similarity index 100% rename from docs/03_guides/code_examples/deployment/apify/proxy_advanced_example.py rename to docs/05_deployment/code_examples/apify/proxy_advanced_example.py diff --git a/docs/03_guides/code_examples/deployment/apify/proxy_example.py b/docs/05_deployment/code_examples/apify/proxy_example.py similarity index 100% rename from docs/03_guides/code_examples/deployment/apify/proxy_example.py rename to docs/05_deployment/code_examples/apify/proxy_example.py diff --git a/docs/03_guides/code_examples/deployment/aws/beautifulsoup_crawler_lambda.py b/docs/05_deployment/code_examples/aws/beautifulsoup_crawler_lambda.py similarity index 100% rename from docs/03_guides/code_examples/deployment/aws/beautifulsoup_crawler_lambda.py rename to docs/05_deployment/code_examples/aws/beautifulsoup_crawler_lambda.py diff --git a/docs/03_guides/code_examples/deployment/aws/playwright_crawler_lambda.py b/docs/05_deployment/code_examples/aws/playwright_crawler_lambda.py similarity index 100% rename from docs/03_guides/code_examples/deployment/aws/playwright_crawler_lambda.py rename to docs/05_deployment/code_examples/aws/playwright_crawler_lambda.py diff --git a/docs/03_guides/code_examples/deployment/aws/playwright_dockerfile b/docs/05_deployment/code_examples/aws/playwright_dockerfile similarity index 100% rename from docs/03_guides/code_examples/deployment/aws/playwright_dockerfile rename to docs/05_deployment/code_examples/aws/playwright_dockerfile diff --git a/docs/03_guides/code_examples/deployment/google/cloud_run_example.py b/docs/05_deployment/code_examples/google/cloud_run_example.py similarity index 100% rename from docs/03_guides/code_examples/deployment/google/cloud_run_example.py rename to docs/05_deployment/code_examples/google/cloud_run_example.py diff --git a/docs/03_guides/code_examples/deployment/google/google_example.py b/docs/05_deployment/code_examples/google/google_example.py similarity index 100% rename from docs/03_guides/code_examples/deployment/google/google_example.py rename to docs/05_deployment/code_examples/google/google_example.py diff --git a/docs/03_guides/15_deploy_on_gcp_cloud_run_functions.mdx b/docs/05_deployment/google_cloud.mdx similarity index 92% rename from docs/03_guides/15_deploy_on_gcp_cloud_run_functions.mdx rename to docs/05_deployment/google_cloud.mdx index aad575ce06..5c4a3535cf 100644 --- a/docs/03_guides/15_deploy_on_gcp_cloud_run_functions.mdx +++ b/docs/05_deployment/google_cloud.mdx @@ -1,6 +1,6 @@ --- -id: deploy-on-gcp-cloud-run-functions -title: Deploying to Google Cloud Run functions +id: gcp-cloud-run-functions +title: Cloud Run functions description: Prepare your crawler to run in Cloud Run functions on Google Cloud Platform. --- @@ -8,7 +8,7 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import GoogleFunctions from '!!raw-loader!./code_examples/deployment/google/google_example.py'; +import GoogleFunctions from '!!raw-loader!./code_examples/google/google_example.py'; [Google Cloud Run Functions](https://cloud.google.com/functions) is a serverless execution environment for running simple HTTP-based web scrapers. This service is best suited for lightweight crawlers that don't require browser rendering capabilities and can be executed via HTTP requests. diff --git a/docs/03_guides/16_deploy_on_gcp_cloud_run.mdx b/docs/05_deployment/google_cloud_run.mdx similarity index 91% rename from docs/03_guides/16_deploy_on_gcp_cloud_run.mdx rename to docs/05_deployment/google_cloud_run.mdx index dcb2f17688..b0e206b03c 100644 --- a/docs/03_guides/16_deploy_on_gcp_cloud_run.mdx +++ b/docs/05_deployment/google_cloud_run.mdx @@ -1,6 +1,6 @@ --- -id: deploy-on-gcp-cloud-run -title: Deploying to Google Cloud Run +id: gcp-cloud-run +title: Cloud Run description: Prepare your crawler to run in Cloud Run on Google Cloud Platform. --- @@ -8,10 +8,10 @@ import ApiLink from '@site/src/components/ApiLink'; import CodeBlock from '@theme/CodeBlock'; -import GoogleCloudRun from '!!raw-loader!./code_examples/deployment/google/cloud_run_example.py'; +import GoogleCloudRun from '!!raw-loader!./code_examples/google/cloud_run_example.py'; -[Google Cloud Run](https://cloud.google.com/run) is a container-based serverless platform that allows you to run web crawlers with headless browsers. This service is recommended when your Crawlee applications need browser rendering capabilities, require more granular control, or have complex dependencies that aren't supported by [Cloud Functions](./deploy-on-gcp-cloud-run-functions). +[Google Cloud Run](https://cloud.google.com/run) is a container-based serverless platform that allows you to run web crawlers with headless browsers. This service is recommended when your Crawlee applications need browser rendering capabilities, require more granular control, or have complex dependencies that aren't supported by [Cloud Functions](./gcp-cloud-run-functions). GCP Cloud Run allows you to deploy using Docker containers, giving you full control over your environment and the flexibility to use any web server framework of your choice, unlike Cloud Functions which are limited to [Flask](https://flask.palletsprojects.com/en/stable/). diff --git a/docs/04_upgrading/upgrading_to_v0x.md b/docs/06_upgrading/upgrading_to_v0x.md similarity index 100% rename from docs/04_upgrading/upgrading_to_v0x.md rename to docs/06_upgrading/upgrading_to_v0x.md diff --git a/docs/04_upgrading/upgrading_to_v1.md b/docs/06_upgrading/upgrading_to_v1.md similarity index 100% rename from docs/04_upgrading/upgrading_to_v1.md rename to docs/06_upgrading/upgrading_to_v1.md diff --git a/pyproject.toml b/pyproject.toml index 4d9ae7e2b4..892ce7ea67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -237,16 +237,16 @@ indent-style = "space" "N999", # Invalid module name "T201", # `print` found ] -"**/docs/02_concepts/code_examples/error_handling/*crawler_with_error_snapshotter.py" = [ +"**/docs/03_concepts/code_examples/error_handling/*crawler_with_error_snapshotter.py" = [ "PLR2004", # Magic value used in comparison. Ignored for simplicity and readability of example code. ] -"**/docs/03_guides/code_examples/running_in_web_server/server.py" = [ +"**/docs/04_guides/code_examples/running_in_web_server/server.py" = [ "TC002", # ruff false positive. Import actually needed during runtime. ] -"**/docs/03_guides/code_examples/creating_web_archive/*.*" = [ +"**/docs/04_guides/code_examples/creating_web_archive/*.*" = [ "ASYNC230", # Ignore for simplicity of the example. ] -"**/docs/03_guides/code_examples/scrapy_migration/scrapy_*.py" = [ +"**/docs/04_guides/code_examples/scrapy_migration/scrapy_*.py" = [ "ANN", # Idiomatic Scrapy tutorial snippets are untyped; shown for comparison, not executed. "RUF012", # Scrapy declares `start_urls` as a plain class-level list. ] @@ -295,7 +295,7 @@ python-version = "3.10" include = ["src", "tests", "scripts", "docs", "website"] exclude = [ "src/crawlee/project_template", - "docs/02_concepts/code_examples/storage_clients/custom_storage_client_example.py", + "docs/03_concepts/code_examples/storage_clients/custom_storage_client_example.py", "website/versioned_docs", ] diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 24d45460ca..a7a2a07ca4 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -191,19 +191,17 @@ module.exports = { 'examples/playwright-crawler-with-fingerprint-generator', ], 'guides/logging-in-with-a-crawler': ['examples/using_browser_profile'], - 'guides/deploy-on-apify': ['deployment/apify-platform'], - 'guides/deploy-on-aws-lambda': ['deployment/aws-lambda'], - 'guides/deploy-on-gcp-cloud-run-functions': ['deployment/gcp-cloud-run-functions'], - 'guides/deploy-on-gcp-cloud-run': ['deployment/gcp-cloud-run'], - guides: ['examples', 'deployment'], + guides: ['examples'], }; - // Doc versions that still have the pre-restructure layout (no numbered concepts - // directory). Their routes (e.g. /docs/1.11/examples/...) must not get redirects - // generated over them. + // Doc versions that still have the pre-restructure layout (no concepts directory). + // Their routes (e.g. /docs/1.11/examples/...) must not get redirects generated + // over them. const versions = require('./versions.json'); const hasOldStructureOnDisk = (version) => - !fs.existsSync(path.join(__dirname, 'versioned_docs', `version-${version}`, '02_concepts')); + !fs + .readdirSync(path.join(__dirname, 'versioned_docs', `version-${version}`)) + .some((dir) => /^\d+_concepts$/.test(dir)); const OLD_STRUCTURE_VERSIONS = versions.filter(hasOldStructureOnDisk); const latestHasOldStructure = OLD_STRUCTURE_VERSIONS.includes(versions[0]); @@ -366,7 +364,7 @@ module.exports = { items: [ { type: 'doc', - docId: 'introduction/introduction', + docId: 'quick-start/quick-start', label: 'Docs', position: 'left', }, diff --git a/website/sidebars.js b/website/sidebars.js index fae4d63e66..cab87b4881 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -1,24 +1,13 @@ module.exports = { docs: [ - { - type: 'doc', - id: 'introduction/introduction', - }, - { - type: 'doc', - id: 'introduction/quick-start', - }, + 'quick-start/quick-start', { type: 'category', - label: 'Tutorial', + label: 'Introduction', collapsed: true, link: { - type: 'generated-index', - title: 'Tutorial', - description: - 'A step-by-step tutorial that takes you from your first crawler to a production-ready scraper for a real website.', - slug: '/introduction/tutorial', - keywords: ['tutorial'], + type: 'doc', + id: 'introduction/introduction', }, items: [ 'introduction/setting-up', @@ -40,14 +29,14 @@ module.exports = { type: 'generated-index', title: 'Concepts', description: - 'How the building blocks of Crawlee work: crawlers, requests, routing, storages, sessions, and more.', + 'Learn how the core components of Crawlee work - crawlers, request routing, storages, sessions, and more.', slug: '/concepts', keywords: ['concepts'], }, items: [ { type: 'autogenerated', - dirName: '02_concepts', + dirName: '03_concepts', }, ], }, @@ -59,14 +48,42 @@ module.exports = { type: 'generated-index', title: 'Guides', description: - 'Step-by-step guides for specific crawling and scraping tasks, integrations, migrations, and deployment.', + 'Task-oriented guides for common crawling and scraping scenarios, integrations, and migrations.', slug: '/guides', keywords: ['guides'], }, items: [ { type: 'autogenerated', - dirName: '03_guides', + dirName: '04_guides', + }, + ], + }, + { + type: 'category', + label: 'Deployment', + collapsed: true, + link: { + type: 'generated-index', + title: 'Deployment guides', + description: 'Here you can find guides on how to deploy your crawlers to various cloud providers.', + slug: '/deployment', + }, + items: [ + { + type: 'doc', + id: 'deployment/apify-platform', + label: 'Deploy on Apify', + }, + { + type: 'doc', + id: 'deployment/aws-lambda', + label: 'Deploy on AWS Lambda', + }, + { + type: 'category', + label: 'Deploy to Google Cloud', + items: ['deployment/gcp-cloud-run-functions', 'deployment/gcp-cloud-run'], }, ], }, @@ -83,7 +100,7 @@ module.exports = { items: [ { type: 'autogenerated', - dirName: '04_upgrading', + dirName: '06_upgrading', }, ], }, From e535eb32beda6e833dcfa03689c79cf3b6b31599 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 13:02:13 +0200 Subject: [PATCH 09/11] docs: group concept pages into sidebar subcategories --- website/sidebars.js | 70 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/website/sidebars.js b/website/sidebars.js index cab87b4881..70621a1a4a 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -1,3 +1,19 @@ +// A collapsible group of concept pages with its own index page at `/concepts/`. The +// description is shown on the group's card in the Concepts index and on the group's index page. +const conceptCategory = (label, slug, description, docIds) => ({ + type: 'category', + label, + description, + collapsed: true, + link: { + type: 'generated-index', + title: label, + description, + slug: `/concepts/${slug}`, + }, + items: docIds.map((docId) => `concepts/${docId}`), +}); + module.exports = { docs: [ 'quick-start/quick-start', @@ -29,15 +45,59 @@ module.exports = { type: 'generated-index', title: 'Concepts', description: - 'Learn how the core components of Crawlee work - crawlers, request routing, storages, sessions, and more.', + 'Learn how the core components of Crawlee work: crawlers, request routing, storages, sessions, and more.', slug: '/concepts', keywords: ['concepts'], }, items: [ - { - type: 'autogenerated', - dirName: '03_concepts', - }, + conceptCategory( + 'Fundamentals', + 'fundamentals', + 'How a crawler processes requests: the architecture, the request object, the crawling context, routing, and error handling.', + ['architecture-overview', 'requests', 'crawling-context', 'request-router', 'error-handling'], + ), + conceptCategory( + 'Crawlers', + 'crawlers', + 'The crawler classes for HTTP, browser, adaptive, and LLM-based crawling, and when to use each one.', + [ + 'http-crawlers', + 'playwright-crawler', + 'adaptive-playwright-crawler', + 'pydantic-ai-crawler', + 'stagehand-crawler', + ], + ), + conceptCategory( + 'Storage', + 'storage', + 'Where crawlers keep data and requests: datasets, key-value stores, request queues, storage backends, and request loaders.', + ['storages', 'storage-clients', 'request-loaders'], + ), + conceptCategory( + 'HTTP and networking', + 'http-and-networking', + 'How crawlers send requests over the network: HTTP clients, headers, sessions, cookies, and proxies.', + ['http-clients', 'http-headers', 'session-management', 'cookie-management', 'proxy-management'], + ), + conceptCategory( + 'Scaling', + 'scaling', + 'How crawlers adjust concurrency to available resources and limit the request rate per domain.', + ['scaling-crawlers', 'request-throttling'], + ), + conceptCategory( + 'Runtime', + 'runtime', + 'How to configure a crawler run and observe it through events, statistics, and logs.', + ['configuration', 'events', 'statistics', 'logging'], + ), + conceptCategory( + 'Advanced', + 'advanced', + 'The service locator and the extension points for plugging your own components into Crawlee.', + ['service-locator', 'extending-crawlee'], + ), ], }, { From 3564b4292d059762e687203dcd4c6c1f20acd9b4 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 13:32:42 +0200 Subject: [PATCH 10/11] docs: correct inaccurate statements in concept pages and guides --- docs/03_concepts/01_architecture_overview.mdx | 2 +- docs/03_concepts/02_requests.mdx | 4 ++-- docs/03_concepts/06_playwright_crawler.mdx | 2 +- docs/03_concepts/08_pydantic_ai_crawler.mdx | 2 +- docs/03_concepts/11_storages.mdx | 6 ++++-- docs/03_concepts/20_error_handling.mdx | 2 +- docs/03_concepts/21_events.mdx | 2 +- docs/03_concepts/22_statistics.mdx | 6 +++--- docs/03_concepts/23_logging.mdx | 4 ++-- docs/03_concepts/24_configuration.mdx | 8 +++++--- docs/03_concepts/25_service_locator.mdx | 2 +- docs/04_guides/02_fill_and_submit_web_form.mdx | 2 +- docs/04_guides/04_avoid_blocking.mdx | 2 +- docs/04_guides/05_respect_robots_txt_file.mdx | 2 +- docs/04_guides/06_stopping_and_resuming.mdx | 2 +- docs/04_guides/07_run_parallel_crawlers.mdx | 2 +- docs/04_guides/11_secure_scraping.mdx | 2 +- 17 files changed, 28 insertions(+), 24 deletions(-) diff --git a/docs/03_concepts/01_architecture_overview.mdx b/docs/03_concepts/01_architecture_overview.mdx index 9c7550a225..0f21ad01cd 100644 --- a/docs/03_concepts/01_architecture_overview.mdx +++ b/docs/03_concepts/01_architecture_overview.mdx @@ -93,7 +93,7 @@ You can learn more about HTTP crawlers on the [HTTP crawlers](./http-crawlers) p Browser crawlers use a real browser to render pages, enabling scraping of sites that require JavaScript. They manage browser instances, pages, and context lifecycles. Crawlee provides two browser crawlers: - `PlaywrightCrawler` utilizes the [Playwright](https://playwright.dev/) library and provides a high-level API for controlling and navigating browsers. You can learn more about it on the [Playwright crawler](./playwright-crawler) page. -- `StagehandCrawler` extends `PlaywrightCrawler` with AI-powered browser automation via [Stagehand](https://github.com/browserbase/stagehand). It adds natural-language methods (`act`, `extract`, `observe`, `execute`) directly on the page object. You can learn more about it in the [Stagehand crawler guide](./stagehand-crawler). +- `StagehandCrawler` extends `PlaywrightCrawler` with AI-powered browser automation via [Stagehand](https://github.com/browserbase/stagehand). It adds natural-language methods (`act`, `extract`, `observe`, `execute`) directly on the page object. You can learn more about it on the [Stagehand crawler page](./stagehand-crawler). ### Adaptive crawler diff --git a/docs/03_concepts/02_requests.mdx b/docs/03_concepts/02_requests.mdx index c8ae914227..b6efcc80da 100644 --- a/docs/03_concepts/02_requests.mdx +++ b/docs/03_concepts/02_requests.mdx @@ -60,7 +60,7 @@ The second request is a duplicate of the first one, so the queue ends up with fi ## Retries -When a request handler raises an exception, the crawler retries the request. It increments `retry_count` and returns the request to the queue. A request is retried until `retry_count` reaches the crawler's `max_request_retries` option, which defaults to 3. To set a different limit for a single request, pass `max_retries` to `from_url`. A request created with `no_retry=True` isn't retried at all, and neither are requests that failed with an HTTP client error status (4xx). Session rotations caused by blocking are counted separately and are limited by `max_session_rotations`. +When a request handler raises an exception, the crawler retries the request. It increments `retry_count` and returns the request to the queue. A request is retried until `retry_count` reaches the crawler's `max_request_retries` option, which defaults to 3. To set a different limit for a single request, pass `max_retries` to `from_url`. A request created with `no_retry=True` isn't retried at all, and neither are requests that failed with an HTTP client error status (4xx), except the status codes that mark a session as blocked (401, 403, and 429 by default), which rotate the session and retry. Session rotations caused by blocking are counted separately and are limited by `max_session_rotations`. When the retries run out, the crawler calls the failed request handler and marks the request as handled. For error handlers, failed request handlers, and the related options, see [Error handling](./error-handling). @@ -84,6 +84,6 @@ Crawlee tracks the progress of a request in its `state` property, a `RequestStat 4. `REQUEST_HANDLER`: the router runs your request handler. 5. `DONE`: the handler finished, and the request is marked as handled. -When an error occurs, the request switches to `ERROR_HANDLER` while the crawler runs the error handler and decides whether to retry. A retried request goes back to the queue and starts again from `BEFORE_NAV`. A request that runs out of retries ends in the `ERROR` state. Requests that the crawler skips, which currently happens only for URLs disallowed by robots.txt, end in the `SKIPPED` state. For details, see [Respecting robots.txt](../guides/respect-robots-txt-file). +When an error occurs, the request switches to `ERROR_HANDLER` while the crawler runs the error handler and decides whether to retry. A retried request goes back to the queue and starts again from `BEFORE_NAV`. A request that runs out of retries ends in the `ERROR` state. Requests skipped because robots.txt disallows them end in the `SKIPPED` state. For details, see [Respecting robots.txt](../guides/respect-robots-txt-file). Failed and skipped requests are marked as handled too, so the queue doesn't hand them out again. The `handled_at` field holds the time a request was marked as handled, and `was_already_handled` tells whether it's set. diff --git a/docs/03_concepts/06_playwright_crawler.mdx b/docs/03_concepts/06_playwright_crawler.mdx index c291de3be4..20c2dff359 100644 --- a/docs/03_concepts/06_playwright_crawler.mdx +++ b/docs/03_concepts/06_playwright_crawler.mdx @@ -32,7 +32,7 @@ If [HTTP-based crawlers](./http-crawlers) are insufficient, {BasicExample} diff --git a/docs/03_concepts/08_pydantic_ai_crawler.mdx b/docs/03_concepts/08_pydantic_ai_crawler.mdx index b2c612b802..c757de661d 100644 --- a/docs/03_concepts/08_pydantic_ai_crawler.mdx +++ b/docs/03_concepts/08_pydantic_ai_crawler.mdx @@ -34,7 +34,7 @@ Use `PydanticAiCrawler` when: For pages with a stable, known structure, a plain `ParselCrawler` or `BeautifulSoupCrawler` is cheaper, since it runs no model calls. -`PydanticAiCrawler` fetches pages over plain HTTP and doesn't render JavaScript. For pages that need a browser, or for complex multi-step interactions, use `StagehandCrawler`. See the [Stagehand crawler guide](./stagehand-crawler). +`PydanticAiCrawler` fetches pages over plain HTTP and doesn't render JavaScript. For pages that need a browser, or for complex multi-step interactions, use `StagehandCrawler`. See [Stagehand crawler](./stagehand-crawler). ## Installation diff --git a/docs/03_concepts/11_storages.mdx b/docs/03_concepts/11_storages.mdx index 0bfa3aec6b..7459e3c769 100644 --- a/docs/03_concepts/11_storages.mdx +++ b/docs/03_concepts/11_storages.mdx @@ -119,7 +119,7 @@ The following code demonstrates the usage of the `RequestQueue`: Requests and data added through the context helpers are written only after the request handler succeeds. For details, see [Crawling context](./crawling-context#results-are-committed-after-the-handler-succeeds). +Crawlee provides helper functions to simplify interactions with the `RequestQueue`. Requests and data added through the context helpers are written only after the request handler succeeds. For details, see [Crawling context](./crawling-context#results-are-committed-after-the-handler-succeeds). The helpers are: - The `add_requests` function allows you to manually add specific URLs to the configured request storage. In this case, you must explicitly provide the URLs you want to be added to the request storage. If you need to specify further details of the request, such as a `label` or `user_data`, you have to pass instances of the `Request` class to the helper. - The `enqueue_links` function is designed to discover new URLs in the current page and add them to the request storage. It can be used with default settings, requiring no arguments, or you can customize its behavior by specifying link element selectors, choosing different enqueue strategies, or applying include/exclude filters to control which URLs are added. See the [Enqueue strategies](../guides/crawling-links#enqueue-strategies) section of the Crawling links guide for more details. @@ -246,12 +246,14 @@ The cleanup occurs as soon as a storage is accessed: ### Disabling automatic purging -To disable automatic purging, set `purge_on_start=False` in your configuration: For setting `purge_on_start` and `storage_dir` in code or through environment variables, see [Configuration](./configuration). +To disable automatic purging, set `purge_on_start=False` in your configuration: {CleaningDoNotPurgeExample} +For setting `purge_on_start` and `storage_dir` in code or through environment variables, see [Configuration](./configuration). + ### Manual purging Purge on start behavior just triggers the storage's `purge` method, which removes all data from the storage. If you want to purge the storage manually, you can do so by calling the `purge` method on the storage instance. Or if you want to delete the storage completely, you can call the `drop` method on the storage instance, which will remove the storage, including metadata and all its data. diff --git a/docs/03_concepts/20_error_handling.mdx b/docs/03_concepts/20_error_handling.mdx index d5abc3a998..0eb47172ad 100644 --- a/docs/03_concepts/20_error_handling.mdx +++ b/docs/03_concepts/20_error_handling.mdx @@ -49,7 +49,7 @@ Here's how to turn off retries for non-network errors using `Statistics` and Crawlee automatically captures a page snapshot on the first occurrence of each unique error. The snapshot can contain an HTML file and a JPEG screenshot of the page where the unhandled exception was raised, and it's saved to the default key-value store. Both `PlaywrightCrawler` and [HTTP crawlers](./http-crawlers) capture the HTML file, but only `PlaywrightCrawler` can capture a screenshot as well. +Debugging a failing request is much easier when you can see what the page looked like at the moment of failure. Set `save_error_snapshots=True` in the crawler's `Statistics` and Crawlee automatically captures a page snapshot on the first occurrence of each unique error. The snapshot can contain an HTML file and a JPEG screenshot of the page where the unhandled exception was raised, and it's saved to the default key-value store, or to the one named by the `persist_state_kvs_name` option of `Statistics`. Both `PlaywrightCrawler` and [HTTP crawlers](./http-crawlers) capture the HTML file, but only `PlaywrightCrawler` can capture a screenshot as well. diff --git a/docs/03_concepts/21_events.mdx b/docs/03_concepts/21_events.mdx index 93bd6562ab..b6a798c7c3 100644 --- a/docs/03_concepts/21_events.mdx +++ b/docs/03_concepts/21_events.mdx @@ -64,7 +64,7 @@ Persisted state only helps if it's still there on the next run. Crawlee purges t The `use_state` helper of the crawling context returns a dictionary that's saved automatically. Every call within a crawler returns the same dictionary, so you can read and update it from any request handler. Crawlee saves it to the default key-value store on every `PERSIST_STATE` event and when the run finishes, and loads it back from there when the crawler starts again with existing storage. The optional argument sets the initial value when no saved state exists yet. -The dictionary is stored under the `CRAWLEE_STATE_` key, where the id is a counter of crawler instances in the process. Pass the same `id` to the constructors of several crawlers to make them share their state. Outside request handlers, use `crawler.use_state()`. Values have to be JSON serializable. +The dictionary is stored under the `CRAWLEE_STATE_` key, where the id is the crawler's `id` argument or, without one, a per-process counter of crawler instances. Pass the same `id` to the constructors of several crawlers to make them share their state. Outside request handlers, use `crawler.use_state()`. Values have to be JSON serializable. The following example stores the page title of every visited URL in the crawler state and reads it after the run: diff --git a/docs/03_concepts/22_statistics.mdx b/docs/03_concepts/22_statistics.mdx index 434ca613ac..aa6c0508ee 100644 --- a/docs/03_concepts/22_statistics.mdx +++ b/docs/03_concepts/22_statistics.mdx @@ -17,7 +17,7 @@ Every crawler collects runtime statistics about the requests it processes. It co The raw data lives in a `StatisticsState` model. Its main fields are: - `requests_finished` and `requests_failed` count requests that succeeded or failed after exhausting their retries. -- `requests_retries` counts requests that needed at least one retry, and `request_retry_histogram` shows how many requests finished after 0, 1, 2, and more retries. +- `requests_retries` counts requests that needed at least one retry, and `request_retry_histogram` shows how many requests finished or failed after 0, 1, 2, and more retries. - `request_total_finished_duration`, `request_total_failed_duration`, `request_min_duration`, and `request_max_duration` measure request processing time. - `requests_with_status_code` maps HTTP status codes to the number of responses that returned them. - `crawler_started_at`, `crawler_finished_at`, and the computed `crawler_runtime` describe the run itself. @@ -37,7 +37,7 @@ The following example reads the summary returned by `run()`, then looks at state {ReadFinalStatisticsExample} -A crawler that you run again with `run()` resets its statistics at the start of the new run. To keep the numbers from a previous run, store the returned `FinalStatistics` before calling `run()` again. +A crawler that you run again with `run()` after a completed run resets its statistics at the start of the new run. To keep the numbers from a previous run, store the returned `FinalStatistics` before calling `run()` again. ## Periodic logging @@ -58,7 +58,7 @@ The following options control persistence: - `persist_state_key` is the key the state is stored under. It defaults to `__CRAWLER_STATISTICS_`, where `` is a per-process counter of `Statistics` instances. - `persist_state_kvs_name` selects a named key-value store. Without it, the default key-value store is used. -The statistics a crawler creates for itself have persistence enabled and use the crawler's key-value store. A `Statistics` instance you create yourself has `persistence_enabled=False` by default. For how to resume an interrupted crawl, see [Stopping and resuming crawlers](../guides/stopping-and-resuming-crawlers). +The statistics a crawler creates for itself have persistence enabled and use the crawler's key-value store. The exception is `AdaptivePlaywrightCrawler`, whose default statistics don't persist. A `Statistics` instance you create yourself has `persistence_enabled=False` by default. For how to resume an interrupted crawl, see [Stopping and resuming crawlers](../guides/stopping-and-resuming-crawlers). ## Custom statistics diff --git a/docs/03_concepts/23_logging.mdx b/docs/03_concepts/23_logging.mdx index d1647cab09..3b01064319 100644 --- a/docs/03_concepts/23_logging.mdx +++ b/docs/03_concepts/23_logging.mdx @@ -9,13 +9,13 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import JsonLoggingExample from '!!raw-loader!roa-loader!./code_examples/logging/configure_json_logging.py'; -Crawlee builds on the standard Python [`logging`](https://docs.python.org/3/library/logging.html) module. Each crawler comes with a logger. HTTP crawlers name it after the crawler class (for example `HttpCrawler`). Inside a request handler the same logger is available as `context.log`, so you can log from your handlers without any setup. +Crawlee builds on the standard Python [`logging`](https://docs.python.org/3/library/logging.html) module. Each crawler comes with a logger. Most HTTP crawlers name it after the crawler class (for example `HttpCrawler`), and the other crawlers after their module (for example `crawlee.crawlers._playwright._playwright_crawler`). Inside a request handler the same logger is available as `context.log`, so you can log from your handlers without any setup. ## Default configuration By default, a crawler sets up the logging infrastructure automatically when it's created. It attaches a handler with a human-readable, colored formatter to the root logger, unless the root logger already has handlers configured. You can opt out by passing `configure_logging=False` to the crawler's constructor (see `BasicCrawlerOptions`) and configure logging yourself. -The log level is controlled by the `Configuration.log_level` option, which can also be set through the `CRAWLEE_LOG_LEVEL` environment variable. For the other settings and the environment variables behind them, see [Configuration](./configuration). +The level of the root logger that the crawler configures comes from the `Configuration.log_level` option of the global configuration, which can also be set through the `CRAWLEE_LOG_LEVEL` environment variable. It has no effect when the crawler doesn't configure logging. For the other settings and the environment variables behind them, see [Configuration](./configuration). ## Statistics logs diff --git a/docs/03_concepts/24_configuration.mdx b/docs/03_concepts/24_configuration.mdx index 2f89904de4..3efce617f8 100644 --- a/docs/03_concepts/24_configuration.mdx +++ b/docs/03_concepts/24_configuration.mdx @@ -34,7 +34,7 @@ The following table lists the options you'll reach for most often. The `Snapshotter` that drives autoscaling, and for `internal_timeout`. +Pass a `Configuration` to the crawler's `configuration` argument to use it for that crawler only. The crawler keeps it in a service locator of its own and uses the configuration for the storages it opens, such as its default request queue and dataset, for the `Snapshotter` that drives autoscaling, and for `internal_timeout`. {CrawlerConfigurationExample} +The crawler still reads the global service locator for the services you don't pass, and some components always read it. If no global configuration is registered yet, creating the crawler registers a default one, so a later `service_locator.set_configuration()` call raises a `ServiceConflictError`. Running the example above logs warnings about this implicit setup. Register any global configuration before you create the crawler. + A few settings are always read from the global configuration, whatever you pass to the crawler: - The log level that the crawler applies when it configures logging. - The browser launch defaults `headless`, `default_browser_path`, and `disable_browser_sandbox`. -- The event intervals, unless you also pass an `event_manager`. Without one, the crawler uses the global event manager and logs a warning about it. +- The event intervals. Autoscaling and state persistence listen to the global event manager, even when you pass an `event_manager` to the crawler. To change these settings, register the configuration globally or configure the component directly, for example with the `headless` argument of `PlaywrightCrawler`. diff --git a/docs/03_concepts/25_service_locator.mdx b/docs/03_concepts/25_service_locator.mdx index c53357cd7c..b585fe5c14 100644 --- a/docs/03_concepts/25_service_locator.mdx +++ b/docs/03_concepts/25_service_locator.mdx @@ -77,7 +77,7 @@ Services can be registered globally through the `ServiceLocator` untouched. Some components always read the global services, for example the log level and the browser launch defaults. For details, see [Configuration](./configuration#applying-a-configuration-to-one-crawler). +Alternatively, services can be passed to the crawler constructors. The crawler keeps them in a service locator of its own, so they apply to that crawler only. For the services you don't pass, the crawler falls back to the global `ServiceLocator`, which creates and registers defaults if none are set. Some components always read the global services, for example the log level, the browser launch defaults, and the event manager that drives autoscaling and state persistence. For details, see [Configuration](./configuration#applying-a-configuration-to-one-crawler). diff --git a/docs/04_guides/02_fill_and_submit_web_form.mdx b/docs/04_guides/02_fill_and_submit_web_form.mdx index 95a917da75..2b5bc8a64b 100644 --- a/docs/04_guides/02_fill_and_submit_web_form.mdx +++ b/docs/04_guides/02_fill_and_submit_web_form.mdx @@ -13,7 +13,7 @@ import RequestExample from '!!raw-loader!roa-loader!./code_examples/fill_and_sub import CrawlerExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form/fill_and_submit_web_form_crawler.py'; import AutomatedExample from '!!raw-loader!roa-loader!./code_examples/fill_and_submit_web_form/fill_and_submit_web_form_automated.py'; -This example demonstrates how to fill and submit a web form using the `HttpCrawler` crawler. The same approach applies to any crawler that inherits from it, such as the `BeautifulSoupCrawler` or `ParselCrawler`. These two crawlers can also [fill in the form automatically](#fill-in-the-form-automatically). +This guide shows how to fill and submit a web form using the `HttpCrawler` crawler. The same approach applies to any crawler that inherits from it, such as the `BeautifulSoupCrawler` or `ParselCrawler`. These two crawlers can also [fill in the form automatically](#fill-in-the-form-automatically). We are going to use the [httpbin.org](https://httpbin.org) website to demonstrate how it works. diff --git a/docs/04_guides/04_avoid_blocking.mdx b/docs/04_guides/04_avoid_blocking.mdx index 2204cd2520..58d37852aa 100644 --- a/docs/04_guides/04_avoid_blocking.mdx +++ b/docs/04_guides/04_avoid_blocking.mdx @@ -42,7 +42,7 @@ Camoufox is an external tool and is not part of Crawlee, so [install it separate :::tip -You can generate a project with the Camoufox integration already set up through the Crawlee CLI. Run `crawlee create` and pick `Playwright-camoufox` when asked for the crawler type. +You can generate a project with the Camoufox integration already set up through the Crawlee CLI. Run `crawlee create` and pick `playwright-camoufox` when asked for the crawler type. ::: diff --git a/docs/04_guides/05_respect_robots_txt_file.mdx b/docs/04_guides/05_respect_robots_txt_file.mdx index 84f2f714d2..650b2d31d1 100644 --- a/docs/04_guides/05_respect_robots_txt_file.mdx +++ b/docs/04_guides/05_respect_robots_txt_file.mdx @@ -10,7 +10,7 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import RespectRobotsTxt from '!!raw-loader!roa-loader!./code_examples/respect_robots_txt_file/respect_robots_txt_file.py'; import OnSkippedRequest from '!!raw-loader!roa-loader!./code_examples/respect_robots_txt_file/respect_robots_on_skipped_request.py'; -This example demonstrates how to configure your crawler to respect the rules established by websites for crawlers as described in the [robots.txt](https://www.robotstxt.org/robotstxt.html) file. +This guide shows how to configure your crawler to respect the rules established by websites for crawlers as described in the [robots.txt](https://www.robotstxt.org/robotstxt.html) file. To configure `Crawlee` to follow the `robots.txt` file, set the parameter `respect_robots_txt_file=True` in `BasicCrawlerOptions`. In this case, `Crawlee` will skip any URLs forbidden in the website's robots.txt file. diff --git a/docs/04_guides/06_stopping_and_resuming.mdx b/docs/04_guides/06_stopping_and_resuming.mdx index 12582d720b..4833122fb5 100644 --- a/docs/04_guides/06_stopping_and_resuming.mdx +++ b/docs/04_guides/06_stopping_and_resuming.mdx @@ -33,7 +33,7 @@ To keep the crawler running even when there are no requests to process at the mo ## Resuming a paused crawl -When a local crawler run is interrupted - say with `CTRL+C` - the default behavior is to start over on the next run, because Crawlee purges the storage on start. To continue from the previous state instead, disable `purge_on_start` in the `Configuration`. For what Crawlee persists and when, see [Events and state persistence](../concepts/events#state-persistence). +When a local crawler run is interrupted - say with `CTRL+C` - the default behavior is to start over on the next run, because Crawlee purges the default storages on start. To continue from the previous state instead, disable `purge_on_start` in the `Configuration`. For what Crawlee persists and when, see [Events and state persistence](../concepts/events#state-persistence). Use the following code and perform two sequential runs. During the first run, stop the crawler by pressing `CTRL+C`, and the second run will resume crawling from where it stopped. diff --git a/docs/04_guides/07_run_parallel_crawlers.mdx b/docs/04_guides/07_run_parallel_crawlers.mdx index 345eca8fd3..0351727a15 100644 --- a/docs/04_guides/07_run_parallel_crawlers.mdx +++ b/docs/04_guides/07_run_parallel_crawlers.mdx @@ -9,7 +9,7 @@ import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import RunParallelCrawlersExample from '!!raw-loader!roa-loader!./code_examples/run_parallel_crawlers/run_parallel_crawlers.py'; -This example demonstrates how to run two parallel crawlers where one crawler processes links discovered by another crawler. +This guide shows how to run two parallel crawlers where one crawler processes links discovered by another crawler. In some situations, you may need different approaches for scraping data from a website. For example, you might use `PlaywrightCrawler` for navigating JavaScript-heavy pages and a faster, more lightweight `ParselCrawler` for processing static pages. One way to solve this is to use `AdaptivePlaywrightCrawler`, see the [Adaptive Playwright crawler](../concepts/adaptive-playwright-crawler) page to learn more. diff --git a/docs/04_guides/11_secure_scraping.mdx b/docs/04_guides/11_secure_scraping.mdx index 97b9b07279..614f0c23cb 100644 --- a/docs/04_guides/11_secure_scraping.mdx +++ b/docs/04_guides/11_secure_scraping.mdx @@ -42,7 +42,7 @@ await context.enqueue_links() await context.enqueue_links(strategy='all') ``` -The same rules apply to URLs read by a `SitemapRequestLoader` and to `Sitemap:` directives in `robots.txt`. A target can't seed your queue with off-host URLs that way either. Following `robots.txt` rules is itself opt-in: `respect_robots_txt_file` is `False` by default. The [respecting robots.txt example](./respect-robots-txt-file) shows how to turn it on. +The same rules apply to URLs read by a `SitemapRequestLoader` and to `Sitemap:` directives in `robots.txt`. A target can't seed your queue with off-host URLs that way either. Following `robots.txt` rules is itself opt-in: `respect_robots_txt_file` is `False` by default. The [respecting robots.txt guide](./respect-robots-txt-file) shows how to turn it on. ### Widening the scope From ab1f6ac35c0237cb10c0b2bfef78bd53f9a0896f Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 16 Oct 2026 13:32:42 +0200 Subject: [PATCH 11/11] chore: compute docs redirect inputs once per build --- website/docusaurus.config.js | 152 ++++++++++++++++------------------- 1 file changed, 71 insertions(+), 81 deletions(-) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index a7a2a07ca4..8064f9b8b1 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -32,6 +32,77 @@ const groupSort = (g1, g2) => { return g1.localeCompare(g2); }; +// Maps a current docs path suffix to the old suffixes that should redirect to it, covering the restructuring that +// introduced the Concepts section and merged the Examples section into Concepts and Guides. Redirects are derived from +// existing routes, so they apply to every docs version that contains the new page and never shadow versions that still +// have the old layout (see OLD_STRUCTURE_VERSIONS below). +const MOVED_DOCS = { + 'concepts/architecture-overview': ['guides/architecture-overview'], + 'concepts/http-crawlers': [ + 'guides/http-crawlers', + 'examples/beautifulsoup-crawler', + 'examples/parsel-crawler', + 'examples/file-download', + ], + 'concepts/playwright-crawler': [ + 'guides/playwright-crawler', + 'examples/playwright-crawler', + 'examples/playwright-crawler-with-block-requests', + 'examples/capture-screenshots-using-playwright', + ], + 'concepts/adaptive-playwright-crawler': [ + 'guides/adaptive-playwright-crawler', + 'examples/adaptive-playwright-crawler', + ], + 'concepts/request-router': ['guides/request-router'], + 'concepts/request-loaders': ['guides/request-loaders', 'examples/using-sitemap-request-loader'], + 'concepts/storages': ['guides/storages', 'examples/add-data-to-dataset', 'examples/export-entire-dataset-to-file'], + 'concepts/storage-clients': ['guides/storage-clients'], + 'concepts/http-clients': ['guides/http-clients'], + 'concepts/session-management': ['guides/session-management'], + 'concepts/cookie-management': ['guides/cookie-management'], + 'concepts/http-headers': ['guides/http-headers'], + 'concepts/proxy-management': ['guides/proxy-management'], + 'concepts/scaling-crawlers': ['guides/scaling-crawlers'], + 'concepts/request-throttling': ['guides/request-throttling'], + 'concepts/error-handling': ['guides/error-handling', 'examples/capturing-page-snapshots-with-error-snapshotter'], + 'concepts/pydantic-ai-crawler': ['guides/pydantic-ai-crawler'], + 'concepts/stagehand-crawler': ['guides/stagehand-crawler'], + 'concepts/logging': ['examples/configure-json-logging'], + 'concepts/service-locator': ['guides/service-locator'], + 'concepts/extending-crawlee': ['guides/extending-crawlee'], + 'guides/crawling-links': [ + 'examples/crawl-all-links-on-website', + 'examples/crawl-multiple-urls', + 'examples/crawl-specific-links-on-website', + 'examples/crawl-website-with-relative-links', + ], + 'guides/fill-and-submit-web-form': ['examples/fill-and-submit-web-form'], + 'guides/respect-robots-txt-file': ['examples/respect-robots-txt-file'], + 'guides/stopping-and-resuming-crawlers': [ + 'examples/crawler-stop', + 'examples/crawler-keep-alive', + 'examples/resuming-paused-crawl', + ], + 'guides/run-parallel-crawlers': ['examples/run-parallel-crawlers'], + 'guides/avoid-blocking': [ + 'examples/playwright-crawler-with-camoufox', + 'examples/playwright-crawler-with-fingerprint-generator', + ], + 'guides/logging-in-with-a-crawler': ['examples/using_browser_profile'], + guides: ['examples'], +}; + +// Doc versions that still have the pre-restructure layout (no concepts directory). Their routes (e.g. +// /docs/1.11/examples/...) must not get redirects generated over them. +const versions = require('./versions.json'); +const hasOldStructureOnDisk = (version) => + !fs + .readdirSync(path.join(__dirname, 'versioned_docs', `version-${version}`)) + .some((dir) => /^\d+_concepts$/.test(dir)); +const OLD_STRUCTURE_VERSIONS = versions.filter(hasOldStructureOnDisk); +const latestHasOldStructure = OLD_STRUCTURE_VERSIONS.includes(versions[0]); + /** @type {Partial} */ module.exports = { title: 'Crawlee for Python ยท Fast, reliable Python web crawlers.', @@ -124,87 +195,6 @@ module.exports = { '@docusaurus/plugin-client-redirects', { createRedirects(existingPath) { - // Maps a current docs path suffix to the old suffixes that should redirect - // to it, covering the restructuring that introduced the Concepts section - // and merged the Examples section into Concepts and Guides. Redirects are - // derived from existing routes, so they apply to every docs version that - // contains the new page and never shadow versions that still have the old - // layout (see OLD_STRUCTURE_VERSIONS below). - const MOVED_DOCS = { - 'concepts/architecture-overview': ['guides/architecture-overview'], - 'concepts/http-crawlers': [ - 'guides/http-crawlers', - 'examples/beautifulsoup-crawler', - 'examples/parsel-crawler', - 'examples/file-download', - ], - 'concepts/playwright-crawler': [ - 'guides/playwright-crawler', - 'examples/playwright-crawler', - 'examples/playwright-crawler-with-block-requests', - 'examples/capture-screenshots-using-playwright', - ], - 'concepts/adaptive-playwright-crawler': [ - 'guides/adaptive-playwright-crawler', - 'examples/adaptive-playwright-crawler', - ], - 'concepts/request-router': ['guides/request-router'], - 'concepts/request-loaders': ['guides/request-loaders', 'examples/using-sitemap-request-loader'], - 'concepts/storages': [ - 'guides/storages', - 'examples/add-data-to-dataset', - 'examples/export-entire-dataset-to-file', - ], - 'concepts/storage-clients': ['guides/storage-clients'], - 'concepts/http-clients': ['guides/http-clients'], - 'concepts/session-management': ['guides/session-management'], - 'concepts/cookie-management': ['guides/cookie-management'], - 'concepts/http-headers': ['guides/http-headers'], - 'concepts/proxy-management': ['guides/proxy-management'], - 'concepts/scaling-crawlers': ['guides/scaling-crawlers'], - 'concepts/request-throttling': ['guides/request-throttling'], - 'concepts/error-handling': [ - 'guides/error-handling', - 'examples/capturing-page-snapshots-with-error-snapshotter', - ], - 'concepts/pydantic-ai-crawler': ['guides/pydantic-ai-crawler'], - 'concepts/stagehand-crawler': ['guides/stagehand-crawler'], - 'concepts/logging': ['examples/configure-json-logging'], - 'concepts/service-locator': ['guides/service-locator'], - 'concepts/extending-crawlee': ['guides/extending-crawlee'], - 'guides/crawling-links': [ - 'examples/crawl-all-links-on-website', - 'examples/crawl-multiple-urls', - 'examples/crawl-specific-links-on-website', - 'examples/crawl-website-with-relative-links', - ], - 'guides/fill-and-submit-web-form': ['examples/fill-and-submit-web-form'], - 'guides/respect-robots-txt-file': ['examples/respect-robots-txt-file'], - 'guides/stopping-and-resuming-crawlers': [ - 'examples/crawler-stop', - 'examples/crawler-keep-alive', - 'examples/resuming-paused-crawl', - ], - 'guides/run-parallel-crawlers': ['examples/run-parallel-crawlers'], - 'guides/avoid-blocking': [ - 'examples/playwright-crawler-with-camoufox', - 'examples/playwright-crawler-with-fingerprint-generator', - ], - 'guides/logging-in-with-a-crawler': ['examples/using_browser_profile'], - guides: ['examples'], - }; - - // Doc versions that still have the pre-restructure layout (no concepts directory). - // Their routes (e.g. /docs/1.11/examples/...) must not get redirects generated - // over them. - const versions = require('./versions.json'); - const hasOldStructureOnDisk = (version) => - !fs - .readdirSync(path.join(__dirname, 'versioned_docs', `version-${version}`)) - .some((dir) => /^\d+_concepts$/.test(dir)); - const OLD_STRUCTURE_VERSIONS = versions.filter(hasOldStructureOnDisk); - const latestHasOldStructure = OLD_STRUCTURE_VERSIONS.includes(versions[0]); - const marker = '/docs/'; const markerIndex = existingPath.indexOf(marker);