tqdm-multiprocess

import time import logging from tqdm_multiprocess import TqdmMultiProcessPool from tqdm import tqdm logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') def worker_function(task_id, sleep_duration, tqdm_func, global_tqdm): # Initialize a worker-specific tqdm bar worker_bar = tqdm_func(total=10, desc=f'Task {task_id}') logging.info(f'Task {task_id}: Starting...') for i in range(10): time.sleep(sleep_duration) # Simulate work worker_bar.update(1) global_tqdm.update(1) # Update the global progress bar as well logging.info(f'Task {task_id}: Finished.') worker_bar.close() return f'Task {task_id} completed' if __name__ == '__main__': # Number of processes to use PROCESS_COUNT = 2 # Total number of small steps across all tasks for the global bar TOTAL_GLOBAL_STEPS = 2 * 10 # 2 tasks * 10 steps each # Initialize a global tqdm bar (optional, can be None) global_pbar = tqdm(total=TOTAL_GLOBAL_STEPS, desc='Global Progress', position=0) # Create a list of tasks: (function_to_run, (args_tuple)) tasks = [ (worker_function, (1, 0.2)), # task_id 1, sleep 0.2s (worker_function, (2, 0.3)) # task_id 2, sleep 0.3s ] # Create the TqdmMultiProcessPool with TqdmMultiProcessPool(PROCESS_COUNT) as pool: # Map tasks to the pool results = pool.map( process_count=PROCESS_COUNT, global_tqdm=global_pbar, task_list=tasks, error_callback=lambda x: logging.error(f'Error: {x}'), done_callback=lambda x: logging.info(f'Done: {x}') ) global_pbar.close() logging.info(f'All tasks completed. Results: {results}')