crawlee

import asyncio from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext import os async def main() -> None: # Initialize the PlaywrightCrawler. # For a real project, consider using persistent storage via storage_client argument. crawler = PlaywrightCrawler( max_requests_per_crawl=10, # Limit for demo purposes headless=True, # Run in headless mode (set to False to see the browser) browser_type='chromium', # Use Chromium browser by default ) # Define the default request handler, which will be called for every request. @crawler.router.default_handler async def request_handler(context: PlaywrightCrawlingContext) -> None: context.log.info(f'Processing {context.request.url} ...') # Extract data from the page using Playwright API. data = { 'url': context.request.url, 'title': await context.page.title(), 'content_length': len(await context.page.content()), } # Push the extracted data to the default dataset. await context.push_data(data) # Extract all links on the page and enqueue them for further crawling. await context.enqueue_links() # Run the crawler with an initial list of URLs. await crawler.run(['https://crawlee.dev']) # Optionally, export the entire dataset to a file (e.g., CSV, JSON). await crawler.export_data('crawlee_results.json') crawler.log.info('Crawling finished. Data exported to crawlee_results.json') if __name__ == '__main__': asyncio.run(main())