� %�g�y� ��ddlZddlZddlZddlZddlmZddlmZddlm Z ddl Z ddl m Z ddl mZmZmZmZmZmZmZmZmZmZmZddlmZdd lmZmZdd lmZd d l m!Z!m"Z"gfd �Z#d�Z$dej%de&e'e'ffd�Z(dej%de)e*e'e&e'e'fffd�Z+dej%de&e'e'ffd�Z,dej%de)e*e'e&e'e'fffd�Z- d dej%de&e'e'fde.de)ej%e&e'e'fffd�Z/de*e'de&e'e'ffd�Z0de dej%de)ej%e&e'e fffd�Z1d�Z2Gd�d��Z3dS)!�N�� literal_eval��which)�Any�)�SageMakerConfig) � DynamoBackend� PrecisionType�is_fp8_available�is_hpu_available�is_ipex_available�is_mlu_available�is_musa_available�is_npu_available�is_sdaa_available�is_torch_xla_available�is_xpu_available)�DEEPSPEED_MULTINODE_LAUNCHERS)�is_port_in_use� merge_dicts)�compare_versions�)�DistributedType�SageMakerDistributedTypec���|�|��\}}t|�����D]9\}}|t|�����vrt |||���:|S)z4 Filters out all `accelerate` specific args )�parse_known_args�vars�items�keys�setattr)�args�parser� default_args�new_args�_�key�values �g/home/asafur/pinokio/api/open-webui.git/app/env/lib/python3.11/site-packages/accelerate/utils/launch.py� _filter_argsr*-su���)�)�,�7�7�K�H�a��4�j�j�&�&�(�(�*�*� ��U� �$�x�.�.�%�%�'�'� '� '� �H�c�5� )� )� )�� �O�c��d�dD��}t|��dkrtd���|d}tj|dg��}d|vr|ddd d fS|d dd d fS)a Determines the executable and argument names for mpirun, based on the type of install. The supported MPI programs are: OpenMPI, Intel MPI, or MVAPICH. Returns: Program name and arg names for hostfile, num processes, and processes per node c�0�g|]}t|���|��S�r)�.0�xs r)� <listcomp>z$_get_mpirun_args.<locals>.<listcomp>@s#��=�=�=�a�E�!�H�H�=��=�=�=r+)�mpirun�mpiexecrz\mpirun or mpiexec were not found. Ensure that Intel MPI, Open MPI, or MVAPICH are installed.z --versionsOpen MPI� --hostfilez-nz --npernodez --bind-toz-fz-ppn�)�len�OSError� subprocess� check_output)�mpi_apps�mpi_app�mpirun_versions r)�_get_mpirun_argsr=8s���>�=�0�=�=�=�H� �8�}�}�����t�u�u�u��q�k�G��,�g�{�-C�D�D�N��n�$�$�� �d�L�+�E�E���d�F�B�.�.r+r"� current_envc�J�d}t|��D]�}|�d��ryt||��}|�g|dkr+|d||dz<|d||dz<|d ||d z<�Ztt||����||�|�����<��|S) z. Setup the FP8 environment variables. � ACCELERATE_�fp8_N�fp8_override_linear_precisionr�FP8_OVERRIDE_FPROPr�FP8_OVERRIDE_DGRADr�FP8_OVERRIDE_WGRAD)r� startswith�getattr�str�upper)r"r>�prefix�argr(s r)� setup_fp8_envrLPs����F��D�z�z� T� T�� �>�>�&� !� !� T��D�#�&�&�E�� ��9�9�9�AF�q��K��)=� =�>�AF�q��K��)=� =�>�AF�q��K��)=� =�>�>�<?���c�@R�@R�<S�<S�K�6� 8�3�9�9�;�;� 8� 8�9�� �r+�returnc�& �g}|jr|jrtd���|j��t ��\}}}}}t |dd��}t |dd��}|j} t |dd��} | r| rt| | z��nd} ||||j|| gz }| r||t| ��gz }|r|||gz }|js;|�tj ��|jr|�d��|�|j ��|� |j ��tj���} t|jp|j��| d <|jrd | d <|jd kr�|j��t+��r |j| d <n�t-��r |j| d<nnt/��r |j| d<nUt1��r |j| d<n<t3��r |j| d<n#t5��r |j| d<n |j| d<|jdkr;|j| d<t|j��| d<|j�t|��| d<n>|jdkr3|j�|jnd| d<|j�t|j��nd| d< t=|j� ����} nL#t$r?td|j� ���dt=j!���d����wxYwt| ��| d<|j� ��dkr-tE��stGd���tI|| ��} tK|j&�'����}nL#t$r?td |j&�'���dtKj!���d����wxYw|j(| d!<|j)| d"<t|j*��| d#<t|j+��| d$<t|j,��| d%<t[��r)t|j.��� ��| d&<|j/rd| d'<|| fS)(zz Prepares and returns the command list and an environment with the correct simple launcher environment variables. �0--module and --no_python cannot be used togetherN� mpirun_cclzbind-to�socket� num_processes�1z-m�ACCELERATE_USE_CPU�true�ACCELERATE_DEBUG_MODE�all�ZE_AFFINITY_MASK�MLU_VISIBLE_DEVICES�SDAA_VISIBLE_DEVICES�MUSA_VISIBLE_DEVICES�ASCEND_RT_VISIBLE_DEVICES�HABANA_VISIBLE_MODULES�CUDA_VISIBLE_DEVICESr� MASTER_ADDR� MASTER_PORT�CCL_WORKER_COUNTz 127.0.0.1�29500�Unknown mixed_precision mode: �. Choose between �.�ACCELERATE_MIXED_PRECISION�fp8�iFP8 is not available on this machine. Please ensure that either Transformer Engine or MSAMP is installed.�Unknown dynamo backend: �ACCELERATE_DYNAMO_BACKEND�ACCELERATE_DYNAMO_MODE�ACCELERATE_DYNAMO_USE_FULLGRAPH�ACCELERATE_DYNAMO_USE_DYNAMIC�OMP_NUM_THREADS�ACCELERATE_USE_IPEX�ACCELERATE_CPU_AFFINITY)0� no_python�module� ValueError�mpirun_hostfiler=rG� num_machinesrH�append�sys� executable�training_script�extend�training_script_args�os�environ�copy�cpu�use_cpu�debug�gpu_idsrrrrrr �main_process_ip�main_process_portrRr �mixed_precision�lower�listr � RuntimeErrorrLr �dynamo_backendrIr(� dynamo_mode�dynamo_use_fullgraph�dynamo_use_dynamic�num_cpu_threads_per_processr�ipex�enable_cpu_affinity)r"�cmd� mpi_app_name� hostfile_arg� num_proc_arg�proc_per_node_arg� bind_to_argrP�bind_torurR�nproc_per_noder>r�r�s r)�prepare_simple_launcher_cmd_envr�bs&�� �C� �~�M�$�+�M��K�L�L�L� ��'�Sc�Se�Se�P� �l�L�2C�[��T�<��6�6� ��$� �8�4�4���(� ���o�t�<�<� �?L�f�Q]�f��]�l�:�;�;�;�cf�� � � � � � � �  � �� � 6� �L�#�m�"4�"4�5� 5�C� � *� �K��)� )�C� �>�� � � �3�>�"�"�"� �;� � �J�J�t� � � ��J�J�t�#�$�$�$��J�J�t�(�)�)�)��*�/�/�#�#�K�(+�D�H�,D�� �(E�(E�K�$�%� �z�6�/5� �+�,� �|�u�����!9� � � � ?�.2�l�K�*� +� +� � � � ?�15��K�-� .� .� � � � ?�26�,�K�.� /� /� � � � ?�26�,�K�.� /� /� � � � ?�7;�|�K�3� 4� 4� � � � ?�48�L�K�0� 1� 1�26�,�K�.� /� ��1���%)�%9� �M�"�%(��)?�%@�%@� �M�"� � � +�.1�*�o�o�K�*� +�� � �a� � �=A�=Q�=]�T�%9�%9�cn� �M�"�DH�DZ�Df�S��)?�%@�%@�%@�ls� �M�"� �'��(<�(B�(B�(D�(D�E�E���� � � � �� s�T�-A�-G�-G�-I�-I� s� s�\i�\n�\p�\p� s� s� s� � � � ���� 14�O�0D�0D�K�,�-� ��!�!�#�#�u�,�,��!�!� ��{��� �$�D�+�6�6� � �&�t�':�'@�'@�'B�'B�C�C���� � � � �� l�t�':�'@�'@�'B�'B� l� l�Ub�Ug�Ui�Ui� l� l� l� � � � ����0>�/C�K�+�,�,0�,<�K�(�)�58��9R�5S�5S�K�1�2�36�t�7N�3O�3O�K�/�0�%(��)I�%J�%J�K�!�"����D�-0���^�^�-A�-A�-C�-C� �)�*� ��5�14� �-�.� � � �s�0&K�A L �&N'�'A O0c� �|j}|j}|j}|j}|dkr�t ||z��|_t |��|_t|j��|_ t|dd��r)t |��|_ t |��|_ n7|�d|��|_ n*t |��|_|�t |��|_ |�d}|dkpt|j��dk}|r"t|��rtd|�d ����|jr|jrt%d ���|jrd |_n|jrd |_t&j���}|jrd |d <t|dd��}|dkr�|j�}t1��r||d<nit3��r||d<nUt5��r||d<nAt7��r||d<n-t9��r||d<nt;��r||d<n||d<|j���} tA|��}n5#t$$r(t%d|�dtAj!���d����wxYwt |��|d<|j���dkr-tE��stGd���tI||��} tK|j&�'����} nL#t$$r?t%d|j&�'���dtKj!���d����wxYw| j(|d<|j)|d<t |j*��|d <t |j+��|d!<|j,�rd |d"<|j-r|j.st%d#���t_|d$��rt |j0��nd%|d&<t |j1��|d'<t |j2�����|d(<t |j3�����|d)<t |j4��|d*<|j5�t |j5��|d+<|j6�t |j6��|d,<|j7�t |j7��|d-<|j8�t |j8��|d.<t |j9�����|d/<t |j:�����|d0<t |j-�����|d1<t |j.�����|d2<t |j;�����|d3<|j<rd |d4<t |j=��|d5<|j>r�d6} d |d7<t |j?��|| d8z<t |j@��|| d9z<t |jA��|| d:z<|jB�t |jB��|| d;z<|jC�t |jC��|| d<z<|jD�t |jD��|| d=z<|jE�t |jE��|| d>z<t |jF��|d?<|jGrd%|d@<|S)Az_ Prepares and returns an environment with the correct multi-GPU environment variables. r� same_networkF�:N�<sr�3Tried to launch distributed communication on port `�$`, but another process is utilizing it. Please specify a different port (such as using the `--main_process_port` flag or specifying a different `main_process_port` in your config file) and rerun your script. To automatically use the next open port (on a single node), you can set this to `0`.rOTrUrVr�rWrXrYrZr[r\r]r^rcrdrerfrgrhrirjrkrlrm�ACCELERATE_USE_FSDPzWWhen using `--fsdp_cpu_ram_efficient_loading` set `--fsdp_sync_module_states` to `True`� fsdp_versionrS� FSDP_VERSION�FSDP_SHARDING_STRATEGY�FSDP_RESHARD_AFTER_FORWARD�FSDP_OFFLOAD_PARAMS�FSDP_MIN_NUM_PARAMS�FSDP_AUTO_WRAP_POLICY�FSDP_TRANSFORMER_CLS_TO_WRAP�FSDP_BACKWARD_PREFETCH�FSDP_STATE_DICT_TYPE�FSDP_FORWARD_PREFETCH�FSDP_USE_ORIG_PARAMS�FSDP_CPU_RAM_EFFICIENT_LOADING�FSDP_SYNC_MODULE_STATES�FSDP_ACTIVATION_CHECKPOINTING�ACCELERATE_USE_TP�TP_SIZE� MEGATRON_LM_�ACCELERATE_USE_MEGATRON_LM� TP_DEGREE� PP_DEGREE�GRADIENT_CLIPPING�NUM_MICRO_BATCHES�SEQUENCE_PARALLELISM�RECOMPUTE_ACTIVATIONS�USE_DISTRIBUTED_OPTIMIZERrnrp)HrRrur�r�rHr��nnodes�int� machine_rank� node_rankrG� master_addr� master_port� rdzv_endpointr�ConnectionErrorrrrqrsr|r}r~r�r�rrrrrr r�r�r r�r r�rLr r�rIr(r�r�r��use_fsdp�fsdp_cpu_ram_efficient_loading�fsdp_sync_module_states�hasattrr��fsdp_sharding_strategy�fsdp_reshard_after_forward�fsdp_offload_params�fsdp_min_num_params�fsdp_auto_wrap_policy�"fsdp_transformer_layer_cls_to_wrap�fsdp_backward_prefetch�fsdp_state_dict_type�fsdp_forward_prefetch�fsdp_use_orig_params�fsdp_activation_checkpointing�use_tp�tp_size�use_megatron_lm�megatron_lm_tp_degree�megatron_lm_pp_degree�megatron_lm_gradient_clipping�megatron_lm_num_micro_batches� megatron_lm_sequence_parallelism�!megatron_lm_recompute_activations�%megatron_lm_use_distributed_optimizerr�r�) r"rRrur�r��need_port_checkr>r�r�r�rJs r)�prepare_multi_gpu_envr��s����&�M��$�L��*�O��.���a���!�-�<�"?�@�@����,�'�'�� ��T�.�/�/��� �4��� /� /� J�"�?�3�3�D� �"�#4�5�5�D� � �$3�!I�!I�6G�!I�!I�D� � �!�-�0�0��� � (�"�#4�5�5�D� �� �!��#�a�'�F�3�t�/@�+A�+A�Q�+F�O�� �>�*;�<�<� �� {�BS� {� {� {� � � �  �{��t�~���K�L�L�L� ����� � � �������*�/�/�#�#�K� �z�6�/5� �+�,��d�I�u�-�-�G��%���D�L�4� � � � :�.5�K�*� +� +� � � � :�18�K�-� .� .� � � � :�29�K�.� /� /� � � � :�29�K�.� /� /� � � � :�7>�K�3� 4� 4� � � � :�4;�K�0� 1� 1�29�K�.� /��*�0�0�2�2�O�u�'��8�8���� �u�u�u��s�/�s�s�\i�\n�\p�\p�s�s�s�t�t�t�u����14�O�0D�0D�K�,�-� ��!�!�#�#�u�,�,��!�!� ��{��� �$�D�+�6�6� � �&�t�':�'@�'@�'B�'B�C�C���� � � � �� l�t�':�'@�'@�'B�'B� l� l�Ub�Ug�Ui�Ui� l� l� l� � � � ����0>�/C�K�+�,�,0�,<�K�(�)�58��9R�5S�5S�K�1�2�36�t�7N�3O�3O�K�/�0� �}�g�-3� �)�*� � .� x�t�7S� x��v�w�w� w�@G��n�@]�@]�&f�c�$�*;�&<�&<�&<�cf� �N�#�14�D�4O�0P�0P� �,�-�47��8W�4X�4X�4^�4^�4`�4`� �0�1�-0��1I�-J�-J�-P�-P�-R�-R� �)�*�-0��1I�-J�-J� �)�*� � %� 1�36�t�7Q�3R�3R�K�/� 0� � 2� >�:=�d�>e�:f�:f�K�6� 7� � &� 2�47��8S�4T�4T�K�0� 1� � $� 0�25�d�6O�2P�2P�K�.� /�/2�4�3M�/N�/N�/T�/T�/V�/V� �+�,�.1�$�2K�.L�.L�.R�.R�.T�.T� �*�+�8;�D�<_�8`�8`�8f�8f�8h�8h� �4�5�14�T�5Q�1R�1R�1X�1X�1Z�1Z� �-�.�7:�4�;]�7^�7^�7d�7d�7f�7f� �3�4� �{�3�+1� �'�(�!$�T�\�!2�!2� �I�� �� p���4:� �0�1�,/��0J�,K�,K� �F�[�(�)�,/��0J�,K�,K� �F�[�(�)�47��8Z�4[�4[� �F�0�0�1� � -� 9�8;�D�<^�8_�8_�K��!4�4� 5� � 0� <�;>�t�?d�;e�;e�K��!7�7� 8� � 1� =�<?��@f�<g�<g�K��!8�8� 9� � 5� A�@C�D�Dn�@o�@o�K��!<�<� =�%(��)I�%J�%J�K�!�"� ��5�14� �-�.� �s�7I�2I9�&L�A M c�L�|j}|j}|j}|j}d}|j�t d|_|dk�r+|jt dk�rdg}|�dt|j��g��|jdkrKtddd��rtd ���|�d t|j ��d g��n*|�d d t|j��g��|j �*|�dt|j ��g��nb|j �*|�dt|j ��g��n1|�dt|j|jz��g��|r$|�dt|��g��|�dt|��g��|jr|jrtd���|jr|�d��n|jr|�d��|�|j��|�|j��n�|dkr�|jt dkr�t||z��|_t|��|_t+|j ��|_t/|dd��r)t|��|_t|��|_n7|�d|��|_n*t|��|_|�t|��|_|�d}|dkpt+|j ��dk}|r"t7|��rt9d|�d����|jr|jrtd���|jrd|_n|jrd|_t:j���}|j rd|d<t/|dd ��}|d kr�|j!�}tE��r||d!<nitG��r||d"<nUtI��r||d#<nAtK��r||d$<n-tM��r||d%<ntO��r||d&<n||d'< tQ|j)�*����} nL#t$r?td(|j)�*���d)tQj+���d*����wxYwtYd+t:j-�.d*����|d+<t| ��|d,<|j)�*��d-kr-t_��stad.���tc||��}t|j2���*��|d/<d|d0<|j3�t|j3��|d1<|j4�t|j4��|d2<|j5�)t|j5���*��|d3<|j6�)t|j6���*��|d4<|j7�)t|j7���*��|d5<|j8�)t|j8���*��|d6<|j9�)t|j9���*��|d7<|j:�t|j:��|d8<|j;rd9|d:<|j<�t|j<��|d;<||fS)<zt Prepares and returns the command list and an environment with the correct DeepSpeed environment variables. Nrr� deepspeedr4�nossh�<z0.14.5z+nossh launcher requires DeepSpeed >= 0.14.5z --node_rankz--no_sshz--no_local_rankz --launcherz --excludez --includez --num_gpusz --master_addrz --master_portrOz--modulez --no_pythonr�Fr�r�r�r�TrUrVr�rWrXrYrZr[r\r]r^rcrdre� PYTHONPATHrfrgrh�ACCELERATE_CONFIG_DS_FIELDS�ACCELERATE_USE_DEEPSPEED�ACCELERATE_DEEPSPEED_ZERO_STAGE�&ACCELERATE_GRADIENT_ACCUMULATION_STEPS�ACCELERATE_GRADIENT_CLIPPING�-ACCELERATE_DEEPSPEED_OFFLOAD_OPTIMIZER_DEVICE�)ACCELERATE_DEEPSPEED_OFFLOAD_PARAM_DEVICE�ACCELERATE_DEEPSPEED_ZERO3_INIT�+ACCELERATE_DEEPSPEED_ZERO3_SAVE_16BIT_MODEL� ACCELERATE_DEEPSPEED_CONFIG_FILErSrp�(ACCELERATE_DEEPSPEED_MOE_LAYER_CLS_NAMES)=rRrur�r��deepspeed_multinode_launcherrrzrH�deepspeed_hostfilerrsr��deepspeed_exclusion_filter�deepspeed_inclusion_filterrrrqrvryr{r�r�r�r�rGr�r�r�rr�r|r}r~r�r�rrrrrr r r�r�r��env_var_path_add�path�abspathr r�rL�'deepspeed_fields_from_accelerate_config� zero_stage�gradient_accumulation_steps�gradient_clipping�offload_optimizer_device�offload_param_device�zero3_init_flag�zero3_save_16bit_model�deepspeed_config_filer��deepspeed_moe_layer_cls_names) r"rRrur�r�r�r�r>r�r�s r)�prepare_deepspeed_cmd_envr�Ms����&�M��$�L��*�O��.�� �C� �(�0�,I�!�,L��)��a���D�=�A^�_`�Aa�a�a��m�� � � �L�#�d�&=�">�">�?�@�@�@� � ,�� 7� 7�� �S�(�;�;� P� �!N�O�O�O� �J�J� �s�4�+<�'=�'=�z�J� K� K� K� K� �J�J�)�<��T�=^�9_�9_�`� a� a� a� � *� 6� �J�J����7�8�8�� � � � � � ,� 8� �J�J����7�8�8�� � � � � �J�J� �c�$�*<��@Q�*Q�&R�&R�S� T� T� T� � @� �J�J���_�)=�)=�>� ?� ?� ?� � � �O�S�):�%;�%;�<�=�=�=� �;� &�4�>� &��O�P�P� P� �[� &� �J�J�z� "� "� "� "� �^� &� �J�J�}� %� %� %� � � �4�'�(�(�(� � � �4�,�-�-�-�-� �� � �d�?�C`�ab�Cc�c�c�!�-�<�"?�@�@����,�'�'�� ��T�.�/�/��� �4��� /� /� J�"�?�3�3�D� �"�#4�5�5�D� � �$3�!I�!I�6G�!I�!I�D� � �!�-�0�0��� � (�"�#4�5�5�D� �� �!��#�a�'�F�3�t�/@�+A�+A�Q�+F�O�� �>�*;�<�<� �� {�BS� {� {� {� � � �  �{��t�~���K�L�L�L� ����� � � �������*�/�/�#�#�K� �z�6�/5� �+�,��d�I�u�-�-�G��%���D�L�4� � � � :�.5�K�*� +� +� � � � :�18�K�-� .� .� � � � :�29�K�.� /� /� � � � :�29�K�.� /� /� � � � :�7>�K�3� 4� 4� � � � :�4;�K�0� 1� 1�29�K�.� /� �'��(<�(B�(B�(D�(D�E�E���� � � � �� s�T�-A�-G�-G�-I�-I� s� s�\i�\n�\p�\p� s� s� s� � � � ���� !1��r�w���s�?S�?S� T� T�K� ��03�O�0D�0D�K�,�-� ��!�!�#�#�u�,�,��!�!� ��{��� �$�D�+�6�6� �14�T�5a�1b�1b�1h�1h�1j�1j�K�-�.�.4�K�*�+� ��"�9<�T�_�9M�9M� �5�6� �'�3�@C�D�Dd�@e�@e� �<�=� ��)�69�$�:P�6Q�6Q�6W�6W�6Y�6Y� �2�3� �$�0�GJ�4�Kh�Gi�Gi�Go�Go�Gq�Gq� �C�D� � �,�CF�t�G`�Ca�Ca�Cg�Cg�Ci�Ci� �?�@� ��'�9<�T�=Q�9R�9R�9X�9X�9Z�9Z� �5�6� �"�.�EH��Id�Ee�Ee�Ek�Ek�Em�Em� �A�B� �!�-�:=�d�>X�:Y�:Y� �6�7� ��5�14� �-�.� �)�5�BE�d�Fh�Bi�Bi� �>�?� � � �s �&R(�(A S1F�podc��|jdkr"td���r|jrd|d<nd|d<|jrd|d<|r|j|_|j|_||fS) zY Prepares and returns an environment with the correct TPU environment variables. �bf16T)� check_is_tpurS�XLA_DOWNCAST_BF16� XLA_USE_BF16rUrV)r�r� downcast_bf16r��tpu_vm�vm�tpu_name�tpu)r"r>r�s r)� prepare_tpur �s���  ��v�%�%�*@�d�*S�*S�*S�%� � � .�/2�K�+� ,� ,�*-�K�� '� �z�6�/5� �+�,� �!��+����=��� �� �r+�nargsc�T�t|��dkriSd�}tj��}|�|��\}}t |��D]�\}}|�d��r�d}|dzt|��kr.||dz�d��rt d���nt d���|�|�||�����|�||�����d�|�|��j � ��D��S) Nrc�z� t|��}|dz|krt|��S|S#t$r|cYSwxYw)Nr)�floatr�rs)�ss r)� _infer_typez+_convert_nargs_to_dict.<locals>._infer_type�sS�� ��a���A��A�v��{�{��1�v�v� ��H��� � � ��H�H�H� ���s�&+�+� :�:)�-z--rulSageMaker doesn’t support argparse actions for `store_true` or `store_false`. Please define explicit types)�type)�actionc�@�i|]\}}||dvrt|��n|��S))�True�Falser)r/r'r(s r)� <dictcomp>z*_convert_nargs_to_dict.<locals>.<dictcomp>sF�� � � � �C�� �U�.?�%?�%?�l�5�!�!�!�U� � � r+) r6�argparse�ArgumentParserr� enumeraterFrs� add_argument� parse_args�__dict__r)r rr#r&�unknown�index�argumentrs r)�_convert_nargs_to_dictr!�sh�� �5�z�z�A�~�~�� ����� $� &� &�F��(�(��/�/�J�A�w�$�W�-�-�=�=���x� � � �{� +� +� =��F��q�y�3�w�<�<�'�'��5�1�9�%�0�0��=�=��$�G����� !�C�����~��#�#�H�;�#�?�?�?�?��#�#�H�V�#�<�<�<�� � � �+�+�E�2�2�;�A�A�C�C� � � �r+�sagemaker_configc�N �td��|jtjd<|j�|jtjd<nF|j�0|j�)|jtjd<|jtjd<ntd���tj� |j ��}|sd}tj� |j ��}|� d��std |�d ����td ��t|j��} t!|j�����}nL#t$r?td |j����d t!j���d����wxYw t)|j�����}nL#t$r?td|j����d t)j���d����wxYwdt/|��|j|jt/|j��t/|j��|jjd�}|j���dkr-t;��st=d���t?||��}d}|jt@j!krddddiii}d} |j"��td|j"�d���i} tG|j"��5} tI| ��D]D\} } | dkr� | �%d��} | d�&��| | d<�E ddd��n #1swxYwYtd| ����d}|j'��td|j'�d���g}tG|j'��5} tI| ��D]Y\} } | dkr� | �%d��} | d| d�&��d�}|�(|���Z ddd��n #1swxYwYtd|����td ��|j)|||j*|j+|j,|j-|j.|j/|j0d!||||d"�}|j1�te|j1|��}|| fS)#Nz(Configuring Amazon SageMaker environment�AWS_DEFAULT_REGION� AWS_PROFILE�AWS_ACCESS_KEY_ID�AWS_SECRET_ACCESS_KEYz]You need to provide an aws_access_key_id and aws_secret_access_key when not using aws_profilerez.pyz8Your training script should be a python script and not "�"z'Converting Arguments to HyperparametersrcrdrirU)�ACCELERATE_USE_SAGEMAKERrfrjrkrlrm�%ACCELERATE_SAGEMAKER_DISTRIBUTED_TYPErgrh� smdistributed� dataparallel�enabledTzLoading SageMaker Inputs from z filer� rzLoaded SageMaker Inputs: zLoading SageMaker Metrics from )�Name�RegexzLoaded SageMaker Metrics: zCreating EstimatorF)� image_uri� entry_point� source_dir�role�transformers_version�pytorch_version� py_version� base_job_name�instance_count� instance_type�debugger_hook_config� distribution�hyperparameters� environment�metric_definitions)3�print�regionr|r}�profile�aws_access_key_id�aws_secret_access_keyr7r��dirnamery�basename�endswithrsr!r{r r�r�r�r r�rIrHr(r�r�r��distributed_typer r�rLr� DATA_PARALLEL�sagemaker_inputs_file�openr�split�strip�sagemaker_metrics_filervr1� iam_role_namer5r6r7r8ru�ec2_instance_type�additional_argsr)r"r"r3r2r=r�r�r>r<�sagemaker_inputs�file�i�line�l�sagemaker_metrics� metric_dicts r)�prepare_sagemager_args_inputsrYs��� � 4�5�5�5�'7�'>�B�J�#�$���+�$4�$<�� �=�!�!� � � +��0J�0V�*.�*@�� �&�'�.2�.H�� �*�+�+��u�v�v�v������!5�6�6�J� ��� ��'�"�"�4�#7�8�8�K� � � �� &� &�d��b�T_�b�b�b�c�c�c� � 3�4�4�4�,�T�-F�G�G�O� �'��(<�(B�(B�(D�(D�E�E���� � � � �� s�T�-A�-G�-G�-I�-I� s� s�\i�\n�\p�\p� s� s� s� � � � ����  �&�t�':�'@�'@�'B�'B�C�C���� � � � �� l�t�':�'@�'@�'B�'B� l� l�Ub�Ug�Ui�Ui� l� l� l� � � � ����%+�&)�/�&:�&:�%3�%9�"&�"2�+.�t�/H�+I�+I�),�T�-D�)E�)E�1A�1R�1X���K� ��!�!�#�#�u�,�,��!�!� ��{��� �$�D�+�6�6� ��L��(�,D�,R�R�R�'�.�9�d�:K�)L�M� ����-�9� �\�/?�/U�\�\�\�]�]�]��� �"�8� 9� 9� 6�T�$�T�?�?� 6� 6���4���6�6���J�J�t�$�$��)*�1������ ��1��&�&�  6� 6� 6� 6� 6� 6� 6� 6� 6� 6� 6� 6���� 6� 6� 6� 6� �<�*:�<�<�=�=�=����.�:� �^�0@�0W�^�^�^�_�_�_��� �"�9� :� :� 6�d�$�T�?�?� 6� 6���4���6�6���J�J�t�$�$���a�D��q�T�Z�Z�\�\��� �"�(�(��5�5�5�5� 6� 6� 6� 6� 6� 6� 6� 6� 6� 6� 6� 6���� 6� 6� 6� 6� �>�+<�>�>�?�?�?� � ����%�/�"� � �.� 0� E�+�;�&�1�)�7�*�7�)�;� %�$�*�"�/� � �D�$�'�3��+�;�T�B�B�� �!� !�!s@�&E�A F�&F9�9A H�4AM�M�M�*A*P!�!P%�(P%c���d�tj�|d���d��D��}|�t |����d�|��S)z� Extends a path-based environment variable's value with a new path and returns the updated value. It's up to the caller to set it in os.environ. c�8�g|]}t|��dk�|��S)r)r6)r/�ps r)r1z$env_var_path_add.<locals>.<listcomp>�s$�� R� R� R�1�s�1�v�v�PQ�z�z�Q�z�z�zr+r5r�)r|r}�getrLrvrH�join)� env_var_name� path_to_add�pathss r)r�r��s`�� S� R�� ���|�R�8�8�>�>�s�C�C� R� R� R�E� �L�L��[�!�!�"�"�"� �8�8�E�?�?�r+c� �eZdZdZdd�Zd�ZdS)�PrepareForLaunchai Prepare a function that will launched in a distributed setup. Args: launcher (`Callable`): The function to launch. distributed_type ([`~state.DistributedType`]): The distributed type to prepare for. debug (`bool`, *optional*, defaults to `False`): Whether or not this is a debug launch. �NOFc�J�||_t|��|_||_dS)N)�launcherrrHr�)�selfrfrHr�s r)�__init__zPrepareForLaunch.__init__�s%�� �� � /�0@� A� A����� � � r+c�@�|jr�ttj�d����}tj�d��}t j�d|t j�||��|���n�|j tj tj tj tjtjtjfvr�t#|��tjd<ttj�dd����}ttj�dd ����}t#||z|z��tjd <t#d��tjd <|j|�dS) N� WORLD_SIZE�ACCELERATE_DEBUG_RDV_FILE�gloo)�rank�store� world_size� LOCAL_RANK�NPROCr� NODE_RANKr�RANK� FORK_LAUNCHED)r�r�r|r}r]�torch� distributed�init_process_group� FileStorerHr� MULTI_GPU� MULTI_MLU� MULTI_MUSA� MULTI_NPU� MULTI_XPU� MULTI_CPUrHrf)rgrr"ro�rdv_file�nprocr�s r)�__call__zPrepareForLaunch.__call__�sN�� �:� @��R�Z�^�^�L�9�9�:�:�J��z�~�~�&A�B�B�H� � � 0� 0����'�1�1�(�J�G�G�%� 1� � � � � � "� � %� � %� � &� � %� � %� � %� ' � � �(+�5�z�z�B�J�|� $��� ���w��2�2�3�3�E��B�J�N�N�;��:�:�;�;�I�!$�U�Y�%6��%>�!?�!?�B�J�v� �&)�!�f�f�� �?�#��� �t����r+N)rdF)�__name__� __module__� __qualname__�__doc__rhr�r.r+r)rcrc�sA������ � ����� ����r+rc)F)4rr|r8rw�astr�shutilr�typingrru�commands.config.config_argsr �utilsr r r r rrrrrrr�utils.constantsr� utils.otherrr�utils.versionsr� dataclassesrrr*r=� Namespace�dictrHrL�tupler�r�r�r��boolr r!rYr�rcr.r+r)�<module>r�s������� � � � ����� � � � ������������������� � � � �9�9�9�9�9�9� � � � � � � � � � � � � � � � � � � � � � � � � � �<�;�;�;�;�;�5�5�5�5�5�5�5�5�-�-�-�-�-�-�B�B�B�B�B�B�B�B�-/�����/�/�/�0��*���c�3�h������$]�(�*<�]��t�C�y�RV�WZ�\_�W_�R`�G`�Aa�]�]�]�]�@H�� 2�H�t�C��H�~�H�H�H�H�VM�H�$6�M�5��c��D�QT�VY�QY�N�AZ�;[�M�M�M�M�bHM��� � ��+/��S��>��@D�� �8� �t�C��H�~� -�.�����('�$�s�)�'��S�#�X��'�'�'�'�Tr"�%�r"�-5�-?�r"� �8� �t�C��H�~� -�.�r"�r"�r"�r"�j���+�+�+�+�+�+�+�+�+�+r+
Memory