OpenPath/prepatching_scripts/create_sample_dataset_txt.py

import cv2
import random
from pathlib import Path
from openslide import OpenSlide
import numpy as np

data_root = Path("/data/TCGA")
output_filename = "sample_dataset_30.txt"
patch_size = 224
max_tries = 1000

def hsv(tile_rgb):
    """
    Checks if a given tile has a high concentration of tissue based on an HSV mask.
    """
    tile = np.array(tile_rgb)
    # Convert from RGB to HSV color space
    tile = cv2.cvtColor(tile, cv2.COLOR_RGB2HSV)
    min_ratio = .6

    # Define the color range for tissue in HSV
    lower_bound = np.array([90, 8, 103])
    upper_bound = np.array([180, 255, 255])

    # Create a mask for the specified color range
    mask = cv2.inRange(tile, lower_bound, upper_bound)

    # Calculate the ratio of tissue pixels
    ratio = np.count_nonzero(mask) / mask.size
    
    if ratio > min_ratio:
        return tile_rgb
    else:
        return None

finish = 3072 * 1000000
svs_files = sorted(str(path) for path in data_root.rglob("*.svs"))
if not svs_files:
    raise RuntimeError(f"No SVS files found under {data_root}")

# Open the output file in write mode ('w')
# This will create the file if it doesn't exist or overwrite it if it does.
with open(output_filename, 'w') as f:
    print(f"Starting patch sampling. Output will be saved to {output_filename}")
    print("\nFor our OpenMidnight checkpoint we ran this script until we reached 29 million patches and then manually force-quit the script. You can adjust the 'finish' variable as needed.")
    
    for e in range(0, finish):
        for path in svs_files:
            image = OpenSlide(path)
            
            # Iterate through each level of the slide
            for level in range(0, image.level_count):
                
                # Get dimensions for the current level being processed
                height = image.level_dimensions[0][1]
                width = image.level_dimensions[0][0]
                
                # Ensure dimensions are valid for patch extraction
                if width < patch_size or height < patch_size:
                    continue

                tries = 0
                while True:
                    tries += 1
                    
                    # Randomly select a top-left coordinate for the patch
                    x = random.randint(0, width - patch_size)
                    y = random.randint(0, height - patch_size)
                    
                    # Read the region from the slide
                    patch = image.read_region((x, y), level=level, size=(patch_size, patch_size))
                    
                    # Check if the patch contains enough tissue
                    res = hsv(patch)
                    
                    if res is not None:
                        # If the patch is valid, write its info to the file
                        output_line = f"{path} {x} {y} {level}\n"
                        f.write(output_line)
                        break # Move to the next level/image
                
                    if tries >= max_tries:
                        # If 1000 random patches at this level are invalid, move on
                        break
            image.close()

# Shuffle the collected entries once generation finishes
with open(output_filename, 'r') as f:
    lines = f.readlines()

random.shuffle(lines)

with open(output_filename, 'w') as f:
    f.writelines(lines)

print("Done")