Auto-labeling Visible Object Masks with HSV

# Auto-Labeling & Segmentation Model Training — 3D Printer Bed Object Masks This notebook builds an auto-labeling pipeline for 3D-printer bed timelapse frames (from `mirdain_data.zip`) — first via HSV-threshold segmentation, then via SAM 3 prompted segmentation — to generate per-object binary masks for the printed squares. It then trains and compares conditioned U-Net models (RGB-only vs. RGB+bed-plan conditioning, with/without hue augmentation) to predict object masks directly from frames, and evaluates robustness to camera color shift. **Inputs:** `mirdain_data.zip` (print timelapse frames + bed plan/print layout). **Outputs:** per-frame label masks/overlays (`label_outputs*` folders), trained U-Net checkpoints, and branch-comparison metrics in `results.json`.

import zipfile, os, shutil, glob, json, math, random
import numpy as np
import cv2
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib.colors import hsv_to_rgb
from tqdm.auto import tqdm
import torch
from pathlib import Path

os.makedirs('/home/user/extracted/images', exist_ok=True)

with zipfile.ZipFile('mirdain_data.zip', 'r') as z:
    z.extractall('/home/user/extracted')

frames = sorted(glob.glob('/home/user/extracted/images/*.jpg'))
print(f"Found {len(frames)} frames")
print(f"Sample: {frames[0]}, {frames[-1]}")
…

Found 293 frames
Sample: /home/user/extracted/images/00300.jpg, /home/user/extracted/images/00600.jpg
bed_mask shape: (256, 256, 3), dtype: uint8
bed_mask unique values: [  0 255]
Frame shape: (720, 1280, 3), dtype: uint8

# Explore bed_mask: the print plan (32 objects in BED coordinates)
bed_mask = cv2.imread('/home/user/extracted/bed_mask.png', cv2.IMREAD_GRAYSCALE)
print(f"bed_mask shape: {bed_mask.shape}")

# Find connected components (the 32 objects)
num_labels, labels_im = cv2.connectedComponents(bed_mask)
print(f"Number of connected components (including background 0): {num_labels}")
# Should be 33 (background + 32 objects)

# Show the bed mask
plt.figure(figsize=(12, 12))
plt.imshow(bed_mask, cmap='gray')
plt.title(f'Bed Mask - {num_labels-1} objects (0=background)')
plt.colorbar(label='Object ID')
plt.axis('off')
plt.show()

# Analyze each object
…

bed_mask shape: (256, 256)
Number of connected components (including background 0): 33

Obj#      Area             Centroid
    1      841 (   72.0,    35.0)
    2      841 (  109.0,    35.0)
    3      841 (  146.0,    35.0)
    4      841 (  183.0,    35.0)
    5      841 (  220.0,    35.0)
    6      841 (   35.0,    72.0)
    7      841 (   72.0,    72.0)
    8      841 (  109.0,    72.0)
    9      841 (  146.0,    72.0)
   10      841 (  183.0,    72.0)
   11      841 (  220.0,    72.0)…

# Sample a few frames to understand layout
sample_frames = frames[::60]  # one every 60 frames
fig, axes = plt.subplots(2, 3, figsize=(18, 10))
axes = axes.ravel()

for i, fp in enumerate(sample_frames[:6]):
    img = cv2.imread(fp)
    img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    axes[i].imshow(img_rgb)
    axes[i].set_title(f"Frame {Path(fp).stem}")
    axes[i].axis('off')

plt.tight_layout()
plt.show()

# Deep-dive into HSV characteristics of printed parts vs bed
sample = cv2.imread(frames[0])
hsv = cv2.cvtColor(sample, cv2.COLOR_BGR2HSV)

# Take a center crop to look at printed parts vs bare bed
h_ch, s_ch, v_ch = cv2.split(hsv)

plt.figure(figsize=(18, 10))
plt.subplot(2, 3, 1)
plt.imshow(cv2.cvtColor(sample, cv2.COLOR_BGR2RGB))
plt.title('Original')
plt.axis('off')

plt.subplot(2, 3, 2)
plt.imshow(h_ch, cmap='hsv')
plt.title('Hue')
plt.colorbar()
plt.axis('off')
…

# More careful look at HSV — pick specific printed objects vs bed
# Let's manually identify a few printed squares and bare bed
img = cv2.imread(frames[20])  # pick a frame mid-print
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

plt.figure(figsize=(16, 8))
plt.subplot(1, 2, 1)
plt.imshow(rgb)
plt.title('Frame - pick a printed square area')
# Mark specific points to sample
plt.scatter([400, 600, 800, 200, 1000], [300, 350, 400, 100, 500], c='red', s=50)
plt.axis('off')

plt.subplot(1, 2, 2)
s_ch = hsv[:,:,1]
plt.imshow(s_ch, cmap='gray')
plt.title('Saturation channel')
…

# Systematic HSV analysis across multiple frames
# Goal: find robust saturation threshold to separate printed parts from grey bed

sample_frames_idx = [0, 30, 60, 90, 120, 150]
s_samps = []
v_samps = []
h_samps = []
bg_samp = []

fig, axes = plt.subplots(2, 3, figsize=(18, 8))
for i, idx in enumerate(sample_frames_idx):
    img = cv2.imread(frames[idx])
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    
    s_ch = hsv[:,:,1]
    v_ch = hsv[:,:,2]
    h_ch = hsv[:,:,0]
    
…

# Systematic HSV analysis - save figures to files
os.makedirs('/home/user/analysis', exist_ok=True)

sample_frames_idx = [0, 30, 60, 90, 120, 150]
fig, axes = plt.subplots(2, 3, figsize=(18, 8))
for i, idx in enumerate(sample_frames_idx):
    img = cv2.imread(frames[idx])
    hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
    s_ch = hsv[:,:,1]
    ax = axes[i//3, i%3]
    im = ax.imshow(s_ch, cmap='viridis', vmin=0, vmax=255)
    ax.set_title(f'Frame {Path(frames[idx]).stem} Saturation')
    ax.axis('off')
    plt.colorbar(im, ax=ax, shrink=0.6)
plt.tight_layout()
plt.savefig('/home/user/analysis/saturation_maps.png', dpi=120)
plt.show()
print("Saved saturation_maps.png")
…

import os, glob, zipfile, numpy as np, cv2, matplotlib.pyplot as plt
from pathlib import Path
os.makedirs('/home/user/extracted/images', exist_ok=True)

# Re-extract if needed
if not os.path.exists('/home/user/extracted/images/00300.jpg'):
    with zipfile.ZipFile('mirdain_data.zip', 'r') as z:
        z.extractall('/home/user/extracted')

frames = sorted(glob.glob('/home/user/extracted/images/*.jpg'))
print(f"Found {len(frames)} frames")

os.makedirs('/home/user/analysis', exist_ok=True)

sample_frames_idx = [0, 30, 60, 90, 120, 150]
fig, axes = plt.subplots(2, 3, figsize=(18, 8))
for i, idx in enumerate(sample_frames_idx):
    img = cv2.imread(frames[idx])
…

Saved to outputs (downloadable from the file manager):
  • analysis/saturation_hist.png (45873 bytes)
  • analysis/saturation_maps.png (1453529 bytes)
  • extracted/bed_mask.png (680 bytes)
  • extracted/images/00300.jpg (149565 bytes)
  • extracted/images/00301.jpg (150461 bytes)
  • extracted/images/00302.jpg (150272 bytes)
  • extracted/images/00303.jpg (155531 bytes)
  • extracted/images/00304.jpg (151847 bytes)
  • extracted/images/00305.jpg (147888 bytes)
  • extracted/images/00306.jpg (14…