training errors
Posted: Tue Jun 21, 2022 11:23 am
Hey is there a guide on error messages that I should be thumbing through?
Not sure what happened, I was about 24 hours into a training session when I moved something on my pc and faceswap hung and closed. So I followed the recovery steps in the training guide and everything seems to be fine but another 24 hours later I am getting some errors and not sure what they mean:
Code: Select all
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\dkroker\MiniConda3\envs\faceswap\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "C:\Users\dkroker\faceswap\lib\gui\display.py", line 186, in _on_tab_change
selected_object.on_tab_select()
File "C:\Users\dkroker\faceswap\lib\gui\display_command.py", line 248, in on_tab_select
self._update_page()
File "C:\Users\dkroker\faceswap\lib\gui\display_page.py", line 273, in _update_page
self.load_display()
File "C:\Users\dkroker\faceswap\lib\gui\display_page.py", line 285, in load_display
self.display_item_process()
File "C:\Users\dkroker\faceswap\lib\gui\display_command.py", line 384, in display_item_process
loss_keys = self.display_item.get_loss_keys(Session.session_ids[-1])
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 254, in get_loss_keys
in self._tb_logs.get_loss(session_id=session_id).items()}
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 490, in get_loss
data = self._cache.get_data(idx, "loss")
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 362, in get_data
val = {metric: np.frombuffer(zlib.decompress(data[metric]),
ValueError: cannot reshape array of size 305004 into shape (152506,2)
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\dkroker\MiniConda3\envs\faceswap\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "C:\Users\dkroker\MiniConda3\envs\faceswap\lib\tkinter\__init__.py", line 814, in callit
func(*args)
File "C:\Users\dkroker\faceswap\lib\gui\display_graph.py", line 364, in refresh
self._calcs = self._thread.get_result() # Terminate the LongRunningTask object
File "C:\Users\dkroker\faceswap\lib\gui\utils.py", line 1263, in get_result
raise self.err[1].with_traceback(self.err[2])
File "C:\Users\dkroker\faceswap\lib\gui\utils.py", line 1234, in run
retval = self._target(*self._args, **self._kwargs)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 567, in refresh
self._get_raw()
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 633, in _get_raw
loss_dict = _SESSION.get_loss(self._session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 179, in get_loss
loss_dict = self._tb_logs.get_loss(session_id=session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 489, in get_loss
self._check_cache(idx)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 465, in _check_cache
self._cache_data(session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 451, in _cache_data
parser.cache_events(session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 610, in cache_events
self._cache.cache_data(session_id, data, self._loss_labels, is_live=self._live_data)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 181, in cache_data
self._add_latest_live(session_id, loss, timestamps)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 326, in _add_latest_live
old = np.frombuffer(zlib.decompress(cache[metric]), dtype=dtype).reshape(old_shape)
ValueError: cannot reshape array of size 152496 into shape (152506,)
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\dkroker\MiniConda3\envs\faceswap\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "C:\Users\dkroker\faceswap\lib\gui\display_graph.py", line 364, in refresh
self._calcs = self._thread.get_result() # Terminate the LongRunningTask object
File "C:\Users\dkroker\faceswap\lib\gui\utils.py", line 1263, in get_result
raise self.err[1].with_traceback(self.err[2])
File "C:\Users\dkroker\faceswap\lib\gui\utils.py", line 1234, in run
retval = self._target(*self._args, **self._kwargs)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 567, in refresh
self._get_raw()
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 633, in _get_raw
loss_dict = _SESSION.get_loss(self._session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 179, in get_loss
loss_dict = self._tb_logs.get_loss(session_id=session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 489, in get_loss
self._check_cache(idx)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 465, in _check_cache
self._cache_data(session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 451, in _cache_data
parser.cache_events(session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 610, in cache_events
self._cache.cache_data(session_id, data, self._loss_labels, is_live=self._live_data)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 181, in cache_data
self._add_latest_live(session_id, loss, timestamps)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 326, in _add_latest_live
old = np.frombuffer(zlib.decompress(cache[metric]), dtype=dtype).reshape(old_shape)
ValueError: cannot reshape array of size 152496 into shape (152506,)
Exception in Tkinter callback
Traceback (most recent call last):
File "C:\Users\dkroker\MiniConda3\envs\faceswap\lib\tkinter\__init__.py", line 1892, in __call__
return self.func(*args)
File "C:\Users\dkroker\faceswap\lib\gui\display.py", line 186, in _on_tab_change
selected_object.on_tab_select()
File "C:\Users\dkroker\faceswap\lib\gui\display_command.py", line 248, in on_tab_select
self._update_page()
File "C:\Users\dkroker\faceswap\lib\gui\display_page.py", line 273, in _update_page
self.load_display()
File "C:\Users\dkroker\faceswap\lib\gui\display_page.py", line 285, in load_display
self.display_item_process()
File "C:\Users\dkroker\faceswap\lib\gui\display_command.py", line 384, in display_item_process
loss_keys = self.display_item.get_loss_keys(Session.session_ids[-1])
File "C:\Users\dkroker\faceswap\lib\gui\analysis\stats.py", line 254, in get_loss_keys
in self._tb_logs.get_loss(session_id=session_id).items()}
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 489, in get_loss
self._check_cache(idx)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 465, in _check_cache
self._cache_data(session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 451, in _cache_data
parser.cache_events(session_id)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 610, in cache_events
self._cache.cache_data(session_id, data, self._loss_labels, is_live=self._live_data)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 181, in cache_data
self._add_latest_live(session_id, loss, timestamps)
File "C:\Users\dkroker\faceswap\lib\gui\analysis\event_reader.py", line 326, in _add_latest_live
old = np.frombuffer(zlib.decompress(cache[metric]), dtype=dtype).reshape(old_shape)
ValueError: cannot reshape array of size 305004 into shape (152506,2)[mod]This was meant to be a reply, but it's here now.
This is just a GUI bug relating to the live graph, it does not prevent training, it just stops the graph updating.
However, I have pushed a fix which should prevent this issue, so please update.[/mod]