�
%�g,5 � �Z � d dl Z d dlmZ d dlZd dlZd dlmZ d dlm Z ddl
mZmZm
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ e� � r d dlmc m Z! ddl"m#Z# ddl$m%Z% e#e&� � Z' dde(de)e* d
e)de)de)de+de+de de,de,fd�Z- dd�Z.dde+de,fd�Z/dde+fd�Z0dS )� N)�Path)�
load_model)�
GradScaler� )�
MODEL_NAME�OPTIMIZER_NAME�RNG_STATE_NAME�SAFE_MODEL_NAME�SAFE_WEIGHTS_NAME�SAMPLER_NAME�SCALER_NAME�SCHEDULER_NAME�WEIGHTS_NAME�get_pretty_name�is_cuda_available�is_hpu_available�is_mlu_available�is_musa_available�is_sdaa_available�is_torch_xla_available�is_xpu_available�load�save)�
get_logger)�PartialStateFT�
output_dir�model_states�
optimizers�
schedulers�dataloaders�
process_index�step�scaler�save_on_each_node�safe_serializationc
�,
� t | � � } t |� � D ]z\ }
}| st nt }|
dk r|� dd|
� d�� � }| � |� � }
t
||
|| �� � t � d|
� �� � �{t |� � D ]{\ }
}|� � � }|
dk r
t � d�nt � d|
� d�}| � |� � }t
|||d�� � t � d|� �� � �|t |� � D ]{\ }
}|� � � }|
dk r
t � d�nt � d|
� d�}| � |� � }t
|||d�� � t � d |� �� � �|t |� � D �] \ }
}|
dk r
t � d�nt � d|
� d�}| � |� � }d
dl
m}m} t! |j |� � r7|� � � }t! ||� � rt
|||d�� � t'