warc3-wet-clueweb09

import gzip import os from warc3_wet_clueweb09 import Warc3Record # Create a dummy WET.gz file for demonstration purposes dummy_wet_content = b"WARC/1.0\r\nWARC-Type: wet\r\nWARC-Record-ID: <urn:uuid:1>\r\nContent-Type: text/plain\r\nContent-Length: 21\r\n\r\nHello ClueWeb09 World!\r\nWARC/1.0\r\nWARC-Type: wet\r\nWARC-Record-ID: <urn:uuid:2>\r\nContent-Type: text/plain\r\nContent-Length: 17\r\n\r\nAnother line here\r\n" dummy_filepath = "dummy.wet.gz" with gzip.open(dummy_filepath, "wb") as f: f.write(dummy_wet_content) # Now, parse records from the dummy file try: with gzip.open(dummy_filepath, 'rb') as f: print(f"Reading records from: {dummy_filepath}") for i, record in enumerate(Warc3Record.parse_records(f)): print(f"--- Record {i+1} ---") print(f"WARC-Type: {record.warc_type}") print(f"WARC-Record-ID: {record.warc_record_id}") if record.content: print(f"Content (first 50 chars): {record.content.decode('utf-8', errors='ignore').strip()[:50]}...") except Exception as e: print(f"An error occurred: {e}") finally: # Clean up the dummy file if os.path.exists(dummy_filepath): os.remove(dummy_filepath) print(f"Cleaned up: {dummy_filepath}")