196 lines
14 KiB
Plaintext
196 lines
14 KiB
Plaintext
setting up the environment by loading in conda environment at Thu Sep 4 18:05:51 CDT 2025
|
|
running the olmo labeling job at Thu Sep 4 18:05:52 CDT 2025
|
|
----------------------------------------
|
|
srun job start: Thu Sep 4 18:05:54 CDT 2025
|
|
Job ID: 3301934
|
|
Username: nws8519
|
|
Queue: gengpu
|
|
Account: p32852
|
|
----------------------------------------
|
|
The following variables are not
|
|
guaranteed to be the same in the
|
|
prologue and the job run script
|
|
----------------------------------------
|
|
PATH (in prologue) : /home/nws8519/.conda/envs/olmo/bin:/software/miniconda3/4.12.0/condabin:/home/nws8519/.local/bin:/home/nws8519/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/usr/lpp/mmfs/bin:/hpc/usertools
|
|
WORKDIR is: /home/nws8519
|
|
----------------------------------------
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/accelerate", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
|
|
args.func(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 1222, in launch_command
|
|
multi_gpu_launcher(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 853, in multi_gpu_launcher
|
|
distrib_run.run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
|
result = agent.run()
|
|
^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
|
result = self._invoke_run(role)
|
|
^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run
|
|
self._initialize_workers(self._worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers
|
|
self._rendezvous(worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 500, in _rendezvous
|
|
rdzv_info = spec.rdzv_handler.next_rendezvous()
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/rendezvous/static_tcp_rendezvous.py", line 67, in next_rendezvous
|
|
self._store = TCPStore( # type: ignore[call-arg]
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
torch.distributed.DistNetworkError: The server socket has failed to listen on any local network address. port: 29505, useIpv6: false, code: -98, name: EADDRINUSE, message: address already in use
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/accelerate", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
|
|
args.func(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 1222, in launch_command
|
|
multi_gpu_launcher(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 853, in multi_gpu_launcher
|
|
distrib_run.run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
|
result = agent.run()
|
|
^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
|
result = self._invoke_run(role)
|
|
^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run
|
|
self._initialize_workers(self._worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers
|
|
self._rendezvous(worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 500, in _rendezvous
|
|
rdzv_info = spec.rdzv_handler.next_rendezvous()
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/rendezvous/static_tcp_rendezvous.py", line 67, in next_rendezvous
|
|
self._store = TCPStore( # type: ignore[call-arg]
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
torch.distributed.DistNetworkError: The server socket has failed to listen on any local network address. port: 29505, useIpv6: false, code: -98, name: EADDRINUSE, message: address already in use
|
|
srun: error: qgpu2005: task 0: Exited with exit code 1
|
|
srun: error: qgpu2008: task 2: Exited with exit code 1
|
|
[W904 18:21:24.281870443 socket.cpp:460] [c10d] waitForInput: poll for socket SocketImpl(fd=27, addr=[qgpu2005]:38060, remote=[qgpu2005]:29505) returned 0, likely a timeout
|
|
[W904 18:21:24.282308265 socket.cpp:485] [c10d] waitForInput: socket SocketImpl(fd=27, addr=[qgpu2005]:38060, remote=[qgpu2005]:29505) timed out after 900000ms
|
|
[W904 18:21:24.731952663 socket.cpp:460] [c10d] waitForInput: poll for socket SocketImpl(fd=27, addr=[qgpu2008]:35800, remote=[qgpu2005]:29505) returned 0, likely a timeout
|
|
[W904 18:21:24.733301968 socket.cpp:485] [c10d] waitForInput: socket SocketImpl(fd=27, addr=[qgpu2008]:35800, remote=[qgpu2005]:29505) timed out after 900000ms
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/accelerate", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
|
|
args.func(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 1222, in launch_command
|
|
multi_gpu_launcher(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 853, in multi_gpu_launcher
|
|
distrib_run.run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
|
result = agent.run()
|
|
^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
|
result = self._invoke_run(role)
|
|
^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run
|
|
self._initialize_workers(self._worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers
|
|
self._rendezvous(worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 513, in _rendezvous
|
|
workers = self._assign_worker_ranks(
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 605, in _assign_worker_ranks
|
|
role_infos_bytes = store.multi_get(
|
|
^^^^^^^^^^^^^^^^
|
|
torch.distributed.DistStoreError: wait timeout after 900000ms, keys: /none/torchelastic/role_info/0, /none/torchelastic/role_info/1
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/accelerate", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/accelerate_cli.py", line 50, in main
|
|
args.func(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 1222, in launch_command
|
|
multi_gpu_launcher(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/accelerate/commands/launch.py", line 853, in multi_gpu_launcher
|
|
distrib_run.run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
|
result = agent.run()
|
|
^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
|
result = self._invoke_run(role)
|
|
^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run
|
|
self._initialize_workers(self._worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers
|
|
self._rendezvous(worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 513, in _rendezvous
|
|
workers = self._assign_worker_ranks(
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 605, in _assign_worker_ranks
|
|
role_infos_bytes = store.multi_get(
|
|
^^^^^^^^^^^^^^^^
|
|
torch.distributed.DistStoreError: wait timeout after 900000ms, keys: /none/torchelastic/role_info/0, /none/torchelastic/role_info/1
|
|
srun: error: qgpu2005: task 1: Exited with exit code 1
|
|
srun: error: qgpu2008: task 3: Exited with exit code 1
|
|
unsupervised olmo categorization pau at Thu Sep 4 18:21:24 CDT 2025
|