Fair enough. Here is the crash log. Ty for your patience as I really should have attached this log.
Code: Select all
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initializing TrainingDataGenerator: (model_input_size: 128, model_output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, color_order: bgr, augment_color: True, no_flip: False, no_warp: False, warp_to_landmarks: False, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initialized TrainingDataGenerator
12/04/2021 02:35:16 MainProcess _training_0 generator minibatch_ab DEBUG Queue batches: (image_count: 558, batchsize: 16, side: 'a', do_shuffle: True, is_preview, False, is_timelapse: False)
12/04/2021 02:35:16 MainProcess _training_0 generator _get_cache DEBUG Creating cache. Side: a
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initializing ImageAugmentation: (batchsize: 16, is_display: False, input_size: 128, output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Output sizes: [256]
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initialized ImageAugmentation
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initializing BackgroundGenerator: (target: '_run', thread_count: 2)
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initialized BackgroundGenerator: '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread(s): '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 1 of 2: '_run_0'
12/04/2021 02:35:16 MainProcess _run_0 generator _minibatch DEBUG Loading minibatch generator: (image_count: 558, side: 'a', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 2 of 2: '_run_1'
12/04/2021 02:35:16 MainProcess _run_1 generator _minibatch DEBUG Loading minibatch generator: (image_count: 558, side: 'a', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Started all threads '_run': 2
12/04/2021 02:35:16 MainProcess _training_0 _base _load_generator DEBUG Loading generator
12/04/2021 02:35:16 MainProcess _training_0 _base _load_generator DEBUG input_size: 128, output_shapes: [(256, 256, 3)]
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initializing TrainingDataGenerator: (model_input_size: 128, model_output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, color_order: bgr, augment_color: True, no_flip: False, no_warp: False, warp_to_landmarks: False, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initialized TrainingDataGenerator
12/04/2021 02:35:16 MainProcess _training_0 generator minibatch_ab DEBUG Queue batches: (image_count: 6059, batchsize: 16, side: 'b', do_shuffle: True, is_preview, False, is_timelapse: False)
12/04/2021 02:35:16 MainProcess _training_0 generator _get_cache DEBUG Creating cache. Side: b
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initializing ImageAugmentation: (batchsize: 16, is_display: False, input_size: 128, output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Output sizes: [256]
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initialized ImageAugmentation
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initializing BackgroundGenerator: (target: '_run', thread_count: 2)
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initialized BackgroundGenerator: '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread(s): '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 1 of 2: '_run_0'
12/04/2021 02:35:16 MainProcess _run_0 generator _minibatch DEBUG Loading minibatch generator: (image_count: 6059, side: 'b', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 2 of 2: '_run_1'
12/04/2021 02:35:16 MainProcess _run_1 generator _minibatch DEBUG Loading minibatch generator: (image_count: 6059, side: 'b', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Started all threads '_run': 2
12/04/2021 02:35:16 MainProcess _training_0 _base _set_preview_feed DEBUG Setting preview feed: (side: 'a')
12/04/2021 02:35:16 MainProcess _training_0 _base _load_generator DEBUG Loading generator
12/04/2021 02:35:16 MainProcess _training_0 _base _load_generator DEBUG input_size: 128, output_shapes: [(256, 256, 3)]
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initializing TrainingDataGenerator: (model_input_size: 128, model_output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, color_order: bgr, augment_color: True, no_flip: False, no_warp: False, warp_to_landmarks: False, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initialized TrainingDataGenerator
12/04/2021 02:35:16 MainProcess _training_0 generator minibatch_ab DEBUG Queue batches: (image_count: 558, batchsize: 14, side: 'a', do_shuffle: True, is_preview, True, is_timelapse: False)
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initializing ImageAugmentation: (batchsize: 14, is_display: True, input_size: 128, output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Output sizes: [256]
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initialized ImageAugmentation
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initializing BackgroundGenerator: (target: '_run', thread_count: 2)
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initialized BackgroundGenerator: '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread(s): '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 1 of 2: '_run_0'
12/04/2021 02:35:16 MainProcess _run_0 generator _minibatch DEBUG Loading minibatch generator: (image_count: 558, side: 'a', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 2 of 2: '_run_1'
12/04/2021 02:35:16 MainProcess _run_1 generator _minibatch DEBUG Loading minibatch generator: (image_count: 558, side: 'a', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Started all threads '_run': 2
12/04/2021 02:35:16 MainProcess _training_0 _base _set_preview_feed DEBUG Setting preview feed: (side: 'b')
12/04/2021 02:35:16 MainProcess _training_0 _base _load_generator DEBUG Loading generator
12/04/2021 02:35:16 MainProcess _training_0 _base _load_generator DEBUG input_size: 128, output_shapes: [(256, 256, 3)]
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initializing TrainingDataGenerator: (model_input_size: 128, model_output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, color_order: bgr, augment_color: True, no_flip: False, no_warp: False, warp_to_landmarks: False, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 generator __init__ DEBUG Initialized TrainingDataGenerator
12/04/2021 02:35:16 MainProcess _training_0 generator minibatch_ab DEBUG Queue batches: (image_count: 6059, batchsize: 14, side: 'b', do_shuffle: True, is_preview, True, is_timelapse: False)
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initializing ImageAugmentation: (batchsize: 14, is_display: True, input_size: 128, output_shapes: [(256, 256, 3)], coverage_ratio: 0.875, config: {'centering': 'face', 'coverage': 87.5, 'icnr_init': False, 'conv_aware_init': False, 'optimizer': 'adam', 'learning_rate': 5e-05, 'epsilon_exponent': -7, 'reflect_padding': False, 'allow_growth': False, 'mixed_precision': False, 'nan_protection': True, 'convert_batchsize': 16, 'loss_function': 'ssim', 'mask_loss_function': 'mse', 'l2_reg_term': 100, 'eye_multiplier': 3, 'mouth_multiplier': 2, 'penalized_mask_loss': True, 'mask_type': 'vgg-obstructed', 'mask_blur_kernel': 3, 'mask_threshold': 4, 'learn_mask': False, 'preview_images': 14, 'zoom_amount': 5, 'rotation_range': 10, 'shift_range': 5, 'flip_chance': 50, 'color_lightness': 30, 'color_ab': 8, 'color_clahe_chance': 50, 'color_clahe_max_size': 4})
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Output sizes: [256]
12/04/2021 02:35:16 MainProcess _training_0 augmentation __init__ DEBUG Initialized ImageAugmentation
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initializing BackgroundGenerator: (target: '_run', thread_count: 2)
12/04/2021 02:35:16 MainProcess _training_0 multithreading __init__ DEBUG Initialized BackgroundGenerator: '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread(s): '_run'
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 1 of 2: '_run_0'
12/04/2021 02:35:16 MainProcess _run_0 generator _minibatch DEBUG Loading minibatch generator: (image_count: 6059, side: 'b', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Starting thread 2 of 2: '_run_1'
12/04/2021 02:35:16 MainProcess _run_1 generator _minibatch DEBUG Loading minibatch generator: (image_count: 6059, side: 'b', do_shuffle: True)
12/04/2021 02:35:16 MainProcess _training_0 multithreading start DEBUG Started all threads '_run': 2
12/04/2021 02:35:16 MainProcess _training_0 _base _set_preview_feed DEBUG Set preview feed. Batchsize: 14
12/04/2021 02:35:16 MainProcess _training_0 _base __init__ DEBUG Initialized _Feeder:
12/04/2021 02:35:16 MainProcess _training_0 _base _set_tensorboard DEBUG Enabling TensorBoard Logging
12/04/2021 02:35:16 MainProcess _training_0 _base _set_tensorboard DEBUG Setting up TensorBoard Logging
12/04/2021 02:35:16 MainProcess _run_0 generator _validate_version DEBUG Setting initial extract version: 2.2
12/04/2021 02:35:16 MainProcess _run_0 augmentation initialize DEBUG Initializing constants. training_size: 96
12/04/2021 02:35:16 MainProcess _run_0 augmentation initialize DEBUG Initialized constants: {'clahe_base_contrast': 0, 'tgt_slices': slice(6, 90, None), 'warp_mapx': '[[[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]]', 'warp_mapy': '[[[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]]', 'warp_pad': 160, 'warp_slices': slice(16, -16, None), 'warp_lm_edge_anchors': '[[[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]]', 'warp_lm_grids': '[[[ 0. 0. 0. ... 0. 0. 0.]\n [ 1. 1. 1. ... 1. 1. 1.]\n [ 2. 2. 2. ... 2. 2. 2.]\n ...\n [93. 93. 93. ... 93. 93. 93.]\n [94. 94. 94. ... 94. 94. 94.]\n [95. 95. 95. ... 95. 95. 95.]]\n\n [[ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n ...\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]]]'}
12/04/2021 02:35:16 MainProcess _run_0 generator _validate_version DEBUG Setting initial extract version: 2.2
12/04/2021 02:35:17 MainProcess _run_0 augmentation initialize DEBUG Initializing constants. training_size: 96
12/04/2021 02:35:17 MainProcess _run_0 augmentation initialize DEBUG Initialized constants: {'clahe_base_contrast': 0, 'tgt_slices': slice(6, 90, None), 'warp_mapx': '[[[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]]', 'warp_mapy': '[[[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]]', 'warp_pad': 160, 'warp_slices': slice(16, -16, None), 'warp_lm_edge_anchors': '[[[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]]', 'warp_lm_grids': '[[[ 0. 0. 0. ... 0. 0. 0.]\n [ 1. 1. 1. ... 1. 1. 1.]\n [ 2. 2. 2. ... 2. 2. 2.]\n ...\n [93. 93. 93. ... 93. 93. 93.]\n [94. 94. 94. ... 94. 94. 94.]\n [95. 95. 95. ... 95. 95. 95.]]\n\n [[ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n ...\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]]]'}
12/04/2021 02:35:17 MainProcess _run_0 multithreading run DEBUG Error in thread (_run_0): Unknown C++ exception from OpenCV code
12/04/2021 02:35:17 MainProcess _run_0 multithreading run DEBUG Error in thread (_run_0): Unknown C++ exception from OpenCV code
12/04/2021 02:35:17 MainProcess _training_0 _base _set_tensorboard VERBOSE Enabled TensorBoard Logging
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initializing _Samples: model: '<plugins.train.model.realface.Model object at 0x0000014668C0E310>', coverage_ratio: 0.875)
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initialized _Samples
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initializing _Timelapse: model: <plugins.train.model.realface.Model object at 0x0000014668C0E310>, coverage_ratio: 0.875, image_count: 14, feeder: '<plugins.train.trainer._base._Feeder object at 0x000001460914C8E0>', image_paths: 2)
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initializing _Samples: model: '<plugins.train.model.realface.Model object at 0x0000014668C0E310>', coverage_ratio: 0.875)
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initialized _Samples
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initialized _Timelapse
12/04/2021 02:35:17 MainProcess _training_0 _base __init__ DEBUG Initialized Trainer
12/04/2021 02:35:17 MainProcess _training_0 train _load_trainer DEBUG Loaded Trainer
12/04/2021 02:35:17 MainProcess _training_0 train _run_training_cycle DEBUG Running Training Cycle
12/04/2021 02:35:17 MainProcess _training_0 multithreading check_and_raise_error DEBUG Thread error caught: [(<class 'cv2.error'>, error('Unknown C++ exception from OpenCV code'), <traceback object at 0x000001465CE58840>)]
12/04/2021 02:35:17 MainProcess _training_0 multithreading run DEBUG Error in thread (_training_0): Unknown C++ exception from OpenCV code
12/04/2021 02:35:17 MainProcess _run_1 multithreading run DEBUG Error in thread (_run_1): Unknown C++ exception from OpenCV code
12/04/2021 02:35:17 MainProcess _run_0 augmentation initialize DEBUG Initializing constants. training_size: 96
12/04/2021 02:35:17 MainProcess _run_0 augmentation initialize DEBUG Initialized constants: {'clahe_base_contrast': 0, 'tgt_slices': slice(6, 90, None), 'warp_mapx': '[[[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]\n\n [[ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]\n [ 6. 27. 48. 69. 90.]]]', 'warp_mapy': '[[[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]\n\n [[ 6. 6. 6. 6. 6.]\n [27. 27. 27. 27. 27.]\n [48. 48. 48. 48. 48.]\n [69. 69. 69. 69. 69.]\n [90. 90. 90. 90. 90.]]]', 'warp_pad': 160, 'warp_slices': slice(16, -16, None), 'warp_lm_edge_anchors': '[[[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]\n\n [[ 0 0]\n [ 0 95]\n [95 95]\n [95 0]\n [47 0]\n [47 95]\n [95 47]\n [ 0 47]]]', 'warp_lm_grids': '[[[ 0. 0. 0. ... 0. 0. 0.]\n [ 1. 1. 1. ... 1. 1. 1.]\n [ 2. 2. 2. ... 2. 2. 2.]\n ...\n [93. 93. 93. ... 93. 93. 93.]\n [94. 94. 94. ... 94. 94. 94.]\n [95. 95. 95. ... 95. 95. 95.]]\n\n [[ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n ...\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]\n [ 0. 1. 2. ... 93. 94. 95.]]]'}
12/04/2021 02:35:17 MainProcess MainThread train _monitor DEBUG Thread error detected
12/04/2021 02:35:17 MainProcess MainThread train _monitor DEBUG Closed Monitor
12/04/2021 02:35:17 MainProcess MainThread train _end_thread DEBUG Ending Training thread
12/04/2021 02:35:17 MainProcess MainThread train _end_thread CRITICAL Error caught! Exiting...
12/04/2021 02:35:17 MainProcess MainThread multithreading join DEBUG Joining Threads: '_training'
12/04/2021 02:35:17 MainProcess MainThread multithreading join DEBUG Joining Thread: '_training_0'
12/04/2021 02:35:17 MainProcess MainThread multithreading join ERROR Caught exception in thread: '_training_0'
Traceback (most recent call last):
File "F:\Programs\faceswap\lib\cli\launcher.py", line 182, in execute_script
process.process()
File "F:\Programs\faceswap\scripts\train.py", line 190, in process
self._end_thread(thread, err)
File "F:\Programs\faceswap\scripts\train.py", line 230, in _end_thread
thread.join()
File "F:\Programs\faceswap\lib\multithreading.py", line 121, in join
raise thread.err[1].with_traceback(thread.err[2])
File "F:\Programs\faceswap\lib\multithreading.py", line 37, in run
self._target(*self._args, **self._kwargs)
File "F:\Programs\faceswap\scripts\train.py", line 252, in _training
raise err
File "F:\Programs\faceswap\scripts\train.py", line 242, in _training
self._run_training_cycle(model, trainer)
File "F:\Programs\faceswap\scripts\train.py", line 327, in _run_training_cycle
trainer.train_one_step(viewer, timelapse)
File "F:\Programs\faceswap\plugins\train\trainer\_base.py", line 191, in train_one_step
model_inputs, model_targets = self._feeder.get_batch()
File "F:\Programs\faceswap\plugins\train\trainer\_base.py", line 416, in get_batch
batch = next(self._feeds[side])
File "F:\Programs\faceswap\lib\multithreading.py", line 156, in iterator
self.check_and_raise_error()
File "F:\Programs\faceswap\lib\multithreading.py", line 84, in check_and_raise_error
raise error[1].with_traceback(error[2])
File "F:\Programs\faceswap\lib\multithreading.py", line 37, in run
self._target(*self._args, **self._kwargs)
File "F:\Programs\faceswap\lib\multithreading.py", line 145, in _run
for item in self.generator(*self._gen_args, **self._gen_kwargs):
File "F:\Programs\faceswap\lib\training\generator.py", line 598, in _minibatch
yield self._process_batch(img_paths, side)
File "F:\Programs\faceswap\lib\training\generator.py", line 643, in _process_batch
batch[..., :3] = self._processing.color_adjust(batch[..., :3])
File "F:\Programs\faceswap\lib\training\augmentation.py", line 225, in color_adjust
batch = self._random_clahe(batch)
File "F:\Programs\faceswap\lib\training\augmentation.py", line 252, in _random_clahe
batch[idx, :, :, 0] = clahe.apply(batch[idx, :, :, 0])
cv2.error: Unknown C++ exception from OpenCV code
============ System Information ============
encoding: cp1252
git_branch: master
git_commits: 8b7b125 Revert "Bugfix -Weights freezing/loading for dfl-sae"
gpu_cuda: No global version found. Check Conda packages for Conda Cuda
gpu_cudnn: No global version found. Check Conda packages for Conda cuDNN
gpu_devices: GPU_0: NVIDIA GeForce RTX 2080
gpu_devices_active: GPU_0
gpu_driver: 496.76
gpu_vram: GPU_0: 8192MB
os_machine: AMD64
os_platform: Windows-10-10.0.19042-SP0
os_release: 10
py_command: F:\Programs\faceswap\faceswap.py train -A F:/Media/Faceswap/Extract Output/Melena 128 Training -B F:/Media/Faceswap/Extract Output/Sandra 128 -m F:/Media/Faceswap/Extract Output/Training Data/128 Melena to Sandra -t realface -bs 16 -it 1000000 -s 150 -ss 25000 -ps 100 -L INFO -gui
py_conda_version: conda 4.10.3
py_implementation: CPython
py_version: 3.8.12
py_virtual_env: True
sys_cores: 16
sys_processor: AMD64 Family 23 Model 8 Stepping 2, AuthenticAMD
sys_ram: Total: 32714MB, Available: 18160MB, Used: 14554MB, Free: 18160MB
=============== Pip Packages ===============
absl-py==0.15.0
astunparse==1.6.3
cachetools==4.2.4
certifi==2021.10.8
charset-normalizer==2.0.7
clang==5.0
cycler==0.10.0
fastcluster==1.1.26
ffmpy==0.2.3
flatbuffers==1.12
gast==0.4.0
google-auth==1.35.0
google-auth-oauthlib==0.4.6
google-pasta==0.2.0
grpcio==1.41.1
h5py==3.1.0
idna==3.3
imageio @ file:///tmp/build/80754af9/imageio_1617700267927/work
imageio-ffmpeg @ file:///home/conda/feedstock_root/build_artifacts/imageio-ffmpeg_1629987409325/work
joblib @ file:///tmp/build/80754af9/joblib_1635411271373/work
keras==2.6.0
Keras-Preprocessing==1.1.2
kiwisolver @ file:///C:/ci/kiwisolver_1612282606037/work
Markdown==3.3.4
matplotlib @ file:///C:/ci/matplotlib-base_1592837548929/work
mkl-fft==1.3.0
mkl-random==1.1.1
mkl-service==2.3.0
numpy @ file:///C:/ci/numpy_and_numpy_base_1603466732592/work
nvidia-ml-py==11.495.46
oauthlib==3.1.1
olefile @ file:///Users/ktietz/demo/mc3/conda-bld/olefile_1629805411829/work
opencv-python==4.5.4.58
opt-einsum==3.3.0
Pillow==8.4.0
protobuf==3.19.1
psutil @ file:///C:/ci/psutil_1612298324802/work
pyasn1==0.4.8
pyasn1-modules==0.2.8
pyparsing @ file:///tmp/build/80754af9/pyparsing_1635766073266/work
python-dateutil @ file:///tmp/build/80754af9/python-dateutil_1626374649649/work
pywin32==228
requests==2.26.0
requests-oauthlib==1.3.0
rsa==4.7.2
scikit-learn @ file:///C:/ci/scikit-learn_1635188126022/work
scipy @ file:///C:/ci/scipy_1616703433439/work
sip==4.19.13
six==1.15.0
tensorboard==2.6.0
tensorboard-data-server==0.6.1
tensorboard-plugin-wit==1.8.0
tensorflow-estimator==2.6.0
tensorflow-gpu==2.6.2
termcolor==1.1.0
threadpoolctl @ file:///Users/ktietz/demo/mc3/conda-bld/threadpoolctl_1629802263681/work
tornado @ file:///C:/ci/tornado_1606942392901/work
tqdm @ file:///tmp/build/80754af9/tqdm_1635330843403/work
typing-extensions==3.7.4.3
urllib3==1.26.7
Werkzeug==2.0.2
wincertstore==0.2
wrapt==1.12.1
============== Conda Packages ==============
# packages in environment at C:\Users\Bean\MiniConda3\envs\faceswap:
#
# Name Version Build Channel
absl-py 0.15.0 pypi_0 pypi
astunparse 1.6.3 pypi_0 pypi
blas 1.0 mkl
ca-certificates 2021.10.8 h5b45459_0 conda-forge
cachetools 4.2.4 pypi_0 pypi
certifi 2021.10.8 py38haa244fe_1 conda-forge
charset-normalizer 2.0.7 pypi_0 pypi
clang 5.0 pypi_0 pypi
cudatoolkit 11.2.2 h933977f_9 conda-forge
cudnn 8.1.0.77 h3e0f4f4_0 conda-forge
cycler 0.10.0 py38_0
fastcluster 1.1.26 py38h251f6bf_2 conda-forge
ffmpeg 4.3.1 ha925a31_0 conda-forge
ffmpy 0.2.3 pypi_0 pypi
flatbuffers 1.12 pypi_0 pypi
freetype 2.10.4 hd328e21_0
gast 0.4.0 pypi_0 pypi
git 2.32.0 haa95532_1
google-auth 1.35.0 pypi_0 pypi
google-auth-oauthlib 0.4.6 pypi_0 pypi
google-pasta 0.2.0 pypi_0 pypi
grpcio 1.41.1 pypi_0 pypi
h5py 3.1.0 pypi_0 pypi
icc_rt 2019.0.0 h0cc432a_1
icu 58.2 ha925a31_3
idna 3.3 pypi_0 pypi
imageio 2.9.0 pyhd3eb1b0_0
imageio-ffmpeg 0.4.5 pyhd8ed1ab_0 conda-forge
intel-openmp 2021.4.0 haa95532_3556
joblib 1.1.0 pyhd3eb1b0_0
jpeg 9d h2bbff1b_0
keras 2.6.0 pypi_0 pypi
keras-preprocessing 1.1.2 pypi_0 pypi
kiwisolver 1.3.1 py38hd77b12b_0
libpng 1.6.37 h2a8f88b_0
libtiff 4.2.0 hd0e1b90_0
libwebp 1.2.0 h2bbff1b_0
lz4-c 1.9.3 h2bbff1b_1
markdown 3.3.4 pypi_0 pypi
matplotlib 3.2.2 0
matplotlib-base 3.2.2 py38h64f37c6_0
mkl 2020.2 256
mkl-service 2.3.0 py38h196d8e1_0
mkl_fft 1.3.0 py38h46781fe_0
mkl_random 1.1.1 py38h47e9c7a_0
numpy 1.19.2 py38hadc3359_0
numpy-base 1.19.2 py38ha3acd2a_0
nvidia-ml-py 11.495.46 pypi_0 pypi
oauthlib 3.1.1 pypi_0 pypi
olefile 0.46 pyhd3eb1b0_0
opencv-python 4.5.4.58 pypi_0 pypi
openssl 1.1.1l h8ffe710_0 conda-forge
opt-einsum 3.3.0 pypi_0 pypi
pillow 8.4.0 py38hd45dc43_0
pip 21.0.1 py38haa95532_0
protobuf 3.19.1 pypi_0 pypi
psutil 5.8.0 py38h2bbff1b_1
pyasn1 0.4.8 pypi_0 pypi
pyasn1-modules 0.2.8 pypi_0 pypi
pyparsing 3.0.4 pyhd3eb1b0_0
pyqt 5.9.2 py38ha925a31_4
python 3.8.12 h6244533_0
python-dateutil 2.8.2 pyhd3eb1b0_0
python_abi 3.8 2_cp38 conda-forge
pywin32 228 py38hbaba5e8_1
qt 5.9.7 vc14h73c81de_0
requests 2.26.0 pypi_0 pypi
requests-oauthlib 1.3.0 pypi_0 pypi
rsa 4.7.2 pypi_0 pypi
scikit-learn 1.0.1 py38hf11a4ad_0
scipy 1.6.2 py38h14eb087_0
setuptools 58.0.4 py38haa95532_0
sip 4.19.13 py38ha925a31_0
six 1.15.0 pypi_0 pypi
sqlite 3.36.0 h2bbff1b_0
tensorboard 2.6.0 pypi_0 pypi
tensorboard-data-server 0.6.1 pypi_0 pypi
tensorboard-plugin-wit 1.8.0 pypi_0 pypi
tensorflow-estimator 2.6.0 pypi_0 pypi
tensorflow-gpu 2.6.2 pypi_0 pypi
termcolor 1.1.0 pypi_0 pypi
threadpoolctl 2.2.0 pyh0d69192_0
tk 8.6.11 h2bbff1b_0
tornado 6.1 py38h2bbff1b_0
tqdm 4.62.3 pyhd3eb1b0_1
typing-extensions 3.7.4.3 pypi_0 pypi
urllib3 1.26.7 pypi_0 pypi
vc 14.2 h21ff451_1
vs2015_runtime 14.27.29016 h5e58377_2
werkzeug 2.0.2 pypi_0 pypi
wheel 0.37.0 pyhd3eb1b0_1
wincertstore 0.2 py38haa95532_2
wrapt 1.12.1 pypi_0 pypi
xz 5.2.5 h62dcd97_0
zlib 1.2.11 h62dcd97_4
zstd 1.4.9 h19a0ad4_0
================= Configs ==================
--------- .faceswap ---------
backend: nvidia
--------- convert.ini ---------
[color.color_transfer]
clip: True
preserve_paper: True
[color.manual_balance]
colorspace: HSV
balance_1: 0.0
balance_2: 0.0
balance_3: 0.0
contrast: 0.0
brightness: 0.0
[color.match_hist]
threshold: 99.0
[mask.box_blend]
type: gaussian
distance: 11.0
radius: 5.0
passes: 1
[mask.mask_blend]
type: normalized
kernel_size: 3
passes: 4
threshold: 4
erosion: 0.0
[scaling.sharpen]
method: none
amount: 150
radius: 0.3
threshold: 5.0
[writer.ffmpeg]
container: mp4
codec: libx264
crf: 23
preset: medium
tune: none
profile: auto
level: auto
skip_mux: False
[writer.gif]
fps: 25
loop: 0
palettesize: 256
subrectangles: False
[writer.opencv]
format: png
draw_transparent: False
jpg_quality: 75
png_compress_level: 3
[writer.pillow]
format: png
draw_transparent: False
optimize: False
gif_interlace: True
jpg_quality: 75
png_compress_level: 3
tif_compression: tiff_deflate
--------- extract.ini ---------
[global]
allow_growth: True
[align.fan]
batch-size: 12
[detect.cv2_dnn]
confidence: 50
[detect.mtcnn]
minsize: 20
scalefactor: 0.709
batch-size: 8
threshold_1: 0.6
threshold_2: 0.7
threshold_3: 0.7
[detect.s3fd]
confidence: 70
batch-size: 4
[mask.bisenet_fp]
batch-size: 8
include_ears: False
include_hair: False
include_glasses: True
[mask.unet_dfl]
batch-size: 8
[mask.vgg_clear]
batch-size: 6
[mask.vgg_obstructed]
batch-size: 2
--------- gui.ini ---------
[global]
fullscreen: False
tab: extract
options_panel_width: 30
console_panel_height: 20
icon_size: 14
font: default
font_size: 9
autosave_last_session: prompt
timeout: 120
auto_load_model_stats: True
--------- train.ini ---------
[global]
centering: face
coverage: 87.5
icnr_init: False
conv_aware_init: False
optimizer: adam
learning_rate: 5e-05
epsilon_exponent: -7
reflect_padding: False
allow_growth: False
mixed_precision: False
nan_protection: True
convert_batchsize: 16
[global.loss]
loss_function: ssim
mask_loss_function: mse
l2_reg_term: 100
eye_multiplier: 3
mouth_multiplier: 2
penalized_mask_loss: True
mask_type: vgg-obstructed
mask_blur_kernel: 3
mask_threshold: 4
learn_mask: False
[model.dfaker]
output_size: 128
[model.dfl_h128]
lowmem: False
[model.dfl_sae]
input_size: 128
clipnorm: True
architecture: df
autoencoder_dims: 0
encoder_dims: 42
decoder_dims: 21
multiscale_decoder: False
[model.dlight]
features: best
details: good
output_size: 256
[model.original]
lowmem: False
[model.phaze_a]
output_size: 128
shared_fc: None
enable_gblock: True
split_fc: True
split_gblock: False
split_decoders: False
enc_architecture: fs_original
enc_scaling: 40
enc_load_weights: True
bottleneck_type: dense
bottleneck_norm: None
bottleneck_size: 1024
bottleneck_in_encoder: True
fc_depth: 1
fc_min_filters: 1024
fc_max_filters: 1024
fc_dimensions: 4
fc_filter_slope: -0.5
fc_dropout: 0.0
fc_upsampler: upsample2d
fc_upsamples: 1
fc_upsample_filters: 512
fc_gblock_depth: 3
fc_gblock_min_nodes: 512
fc_gblock_max_nodes: 512
fc_gblock_filter_slope: -0.5
fc_gblock_dropout: 0.0
dec_upscale_method: subpixel
dec_norm: None
dec_min_filters: 64
dec_max_filters: 512
dec_filter_slope: -0.45
dec_res_blocks: 1
dec_output_kernel: 5
dec_gaussian: True
dec_skip_last_residual: True
freeze_layers: keras_encoder
load_layers: encoder
fs_original_depth: 4
fs_original_min_filters: 128
fs_original_max_filters: 1024
mobilenet_width: 1.0
mobilenet_depth: 1
mobilenet_dropout: 0.001
[model.realface]
input_size: 128
output_size: 256
dense_nodes: 1536
complexity_encoder: 128
complexity_decoder: 512
[model.unbalanced]
input_size: 128
lowmem: False
clipnorm: True
nodes: 1024
complexity_encoder: 128
complexity_decoder_a: 384
complexity_decoder_b: 512
[model.villain]
lowmem: False
[trainer.original]
preview_images: 14
zoom_amount: 5
rotation_range: 10
shift_range: 5
flip_chance: 50
color_lightness: 30
color_ab: 8
color_clahe_chance: 50
color_clahe_max_size: 4