I've been looking around in the forums, but not finding a lot of info on what I can do to improve an existing model's training. I have a model that was trained on two sets of 3000 samples for A and B with plenty of different angles. The training in graph analysis turned pretty much into a straight line at around 0.02 error for A after 50K iterations (B side was 0.03, but I was not looking for swapping B>A anyway).
My data sets were fairly cleaned up manually - not saying they are perfect, as it is my first time at this. I read in some threads here that we can stop training and make some changes to the input data (add/remove), but not sure what I am adding or removing? For example, if I see the preview for training is having difficulty with side views, should I add more sides and remove some frontal views, or just add more side views. (I think I have an OK distribution of all angels in my datasets as-is). Should I change some settings and run the training again (loading weights where it left off)?
Anyway, any pointers/reference is appreciated. Below my config I used for training (pretty much default everything per your awesome tutorials!!!)
One note, during training with batch size of 16, I was running into a WARNING with value of an index going out of range after a few thousands of integrations, so I did some training on 64 batch size and it was OK, then I tried 16 again and got the same warning, so settled for 32 batch size.
Code: Select all
{
"convert": {
"Input Dir": "",
"Output Dir": "",
"Alignments": "",
"Reference Video": "",
"Model Dir": "",
"Color Adjustment": "avg-color",
"Mask Type": "extended",
"Writer": "opencv",
"Output Scale": 100,
"Frame Ranges": "",
"Input Aligned Dir": "",
"Nfilter": "",
"Filter": "",
"Ref Threshold": 0.4,
"Jobs": 0,
"Trainer": "",
"On The Fly": false,
"Keep Unchanged": false,
"Swap Model": false,
"Singleprocess": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"extract": {
"Input Dir": "",
"Output Dir": "",
"Alignments": "",
"Detector": "s3fd",
"Aligner": "fan",
"Masker": "",
"Normalization": "none",
"Re Feed": 0,
"Rotate Images": "",
"Min Size": 0,
"Nfilter": "",
"Filter": "",
"Ref Threshold": 0.4,
"Size": 512,
"Extract Every N": 1,
"Save Interval": 0,
"Debug Landmarks": false,
"Singleprocess": false,
"Skip Existing": false,
"Skip Existing Faces": false,
"Skip Saving Faces": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"train": {
"Input A": "D:/Data/Model A/Trainer",
"Input B": "D:/Data/Model B/Trainer",
"Model Dir": "D:/Data/Model 1",
"Load Weights": "D:/Data/Model 1/dfaker.h5",
"Trainer": "dfaker",
"Summary": false,
"Freeze Weights": false,
"Batch Size": 32,
"Iterations": 1000000,
"Distributed": false,
"Save Interval": 250,
"Snapshot Interval": 25000,
"Timelapse Input A": "",
"Timelapse Input B": "",
"Timelapse Output": "",
"Preview Scale": 100,
"Preview": false,
"Write Image": false,
"No Logs": false,
"Warp To Landmarks": false,
"No Flip": false,
"No Augment Color": false,
"No Warp": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"alignments": {
"Job": "",
"Output": "console",
"Alignments File": "",
"Faces Folder": "",
"Frames Folder": "",
"Extract Every N": 1,
"Size": 512,
"Large": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"effmpeg": {
"Action": "extract",
"Input": "input",
"Output": "",
"Reference Video": "",
"Fps": "-1.0",
"Extract Filetype": ".png",
"Start": "00:00:00",
"End": "00:00:00",
"Duration": "00:00:00",
"Mux Audio": false,
"Transpose": "",
"Degrees": "",
"Scale": "1920x1080",
"Quiet": false,
"Verbose": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"manual": {
"Alignments": "",
"Frames": "",
"Thumb Regen": false,
"Single Process": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"mask": {
"Alignments": "",
"Input": "",
"Input Type": "frames",
"Masker": "extended",
"Processing": "missing",
"Output Folder": "",
"Blur Kernel": 3,
"Threshold": 4,
"Output Type": "combined",
"Full Frame": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"preview": {
"Input Dir": "",
"Alignments": "",
"Model Dir": "",
"Swap Model": false,
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"restore": {
"Model Dir": "",
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"sort": {
"Input": "",
"Output": "",
"Sort By": "face",
"Keep": false,
"Ref Threshold": -1.0,
"Final Process": "rename",
"Group By": "hist",
"Bins": 5,
"Log Changes": false,
"Log File": "sort_log.json",
"Exclude Gpus": "",
"Configfile": "",
"Loglevel": "INFO",
"Logfile": ""
},
"tab_name": "train"
}
Thank you!