robotspy

from robotspy import RobotFileParser import urllib.request # robotspy uses urllib.request internally # Initialize the parser with the URL of the robots.txt file # Using a well-known site's robots.txt for demonstration parser = RobotFileParser(url="https://www.google.com/robots.txt") try: # Read the robots.txt file from the specified URL # This performs an HTTP GET request to fetch the file. parser.read() print(f"User-agent '*' can fetch /: {parser.can_fetch('*', '/')}") print(f"User-agent 'Googlebot' can fetch /search: {parser.can_fetch('Googlebot', '/search')}") print(f"User-agent 'Googlebot' can fetch /images/search: {parser.can_fetch('Googlebot', '/images/search')}") print(f"User-agent 'AdsBot-Google' can fetch /ads: {parser.can_fetch('AdsBot-Google', '/ads')}") except Exception as e: print(f"Error reading robots.txt: {e}") print("Ensure you have network connectivity and the URL is correct.") # Example of initializing with a custom user agent for fetching the robots.txt file itself # This can help avoid 403 errors from some servers. parser_custom_ua = RobotFileParser(url="https://www.google.com/robots.txt", user_agent="MyCustomCrawler/1.0") try: parser_custom_ua.read() print(f"\nUsing 'MyCustomCrawler/1.0' to fetch, then checking for 'Googlebot': {parser_custom_ua.can_fetch('Googlebot', '/')}") except Exception as e: print(f"Error reading robots.txt with custom user agent: {e}")