apify

import asyncio import httpx from bs4 import BeautifulSoup from apify import Actor async def main() -> None: async with Actor: # Retrieve the Actor input, or use a default if not provided actor_input = await Actor.get_input() or {} start_urls = actor_input.get('start_urls', [{'url': 'https://apify.com'}]) # Open the default request queue request_queue = await Actor.open_request_queue() # Enqueue the start URLs for start_url_obj in start_urls: url = start_url_obj.get('url') if url: await request_queue.add_request(url) # Process the URLs from the request queue while True: request = await request_queue.fetch_next_request() if not request: break Actor.log.info(f'Processing {request.url}') try: async with httpx.AsyncClient() as client: response = await client.get(request.url) response.raise_for_status() soup = BeautifulSoup(response.content, 'html.parser') data = { 'url': request.url, 'title': soup.title.string if soup.title else None, 'status_code': response.status_code } await Actor.push_data(data) except httpx.HTTPStatusError as e: Actor.log.error(f'Failed to fetch {request.url}: {e}') finally: await request_queue.mark_request_handled(request) if __name__ == '__main__': asyncio.run(main())