168 lines
11 KiB
Plaintext
168 lines
11 KiB
Plaintext
setting up the environment by loading in conda environment at Wed Sep 3 19:04:03 CDT 2025
|
|
running the bertopic job at Wed Sep 3 19:04:03 CDT 2025
|
|
----------------------------------------
|
|
srun job start: Wed Sep 3 19:04:03 CDT 2025
|
|
Job ID: 3220869
|
|
Username: nws8519
|
|
Queue: gengpu
|
|
Account: p32852
|
|
----------------------------------------
|
|
The following variables are not
|
|
guaranteed to be the same in the
|
|
prologue and the job run script
|
|
----------------------------------------
|
|
PATH (in prologue) : /home/nws8519/.conda/envs/olmo/bin:/software/miniconda3/4.12.0/condabin:/home/nws8519/.local/bin:/home/nws8519/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/usr/lpp/mmfs/bin:/hpc/usertools
|
|
WORKDIR is: /home/nws8519
|
|
----------------------------------------
|
|
/home/nws8519/.conda/envs/olmo/bin/python3.11: can't open file '/gpfs/home/nws8519/git/mw-lifecycle-analysis/p2/quest/nnodes': [Errno 2] No such file or directory
|
|
/home/nws8519/.conda/envs/olmo/bin/python3.11: can't open file '/gpfs/home/nws8519/git/mw-lifecycle-analysis/p2/quest/nnodes': [Errno 2] No such file or directory
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/torchrun", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/torchrun", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
|
return f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in main
|
|
return f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in main
|
|
run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 261, in launch_agent
|
|
result = agent.run()
|
|
result = agent.run()
|
|
^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run
|
|
result = self._invoke_run(role)
|
|
result = self._invoke_run(role)
|
|
^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run
|
|
^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 864, in _invoke_run
|
|
self._initialize_workers(self._worker_group)
|
|
self._initialize_workers(self._worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 683, in _initialize_workers
|
|
self._rendezvous(worker_group)
|
|
self._rendezvous(worker_group)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper
|
|
result = f(*args, **kwargs)
|
|
result = f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 500, in _rendezvous
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/agent/server/api.py", line 500, in _rendezvous
|
|
rdzv_info = spec.rdzv_handler.next_rendezvous()
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/rendezvous/static_tcp_rendezvous.py", line 67, in next_rendezvous
|
|
self._store = TCPStore( # type: ignore[call-arg]
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
torch.distributed.DistNetworkError: The server socket has failed to listen on any local network address. port: 29500, useIpv6: false, code: -98, name: EADDRINUSE, message: address already in use
|
|
rdzv_info = spec.rdzv_handler.next_rendezvous()
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/rendezvous/static_tcp_rendezvous.py", line 67, in next_rendezvous
|
|
self._store = TCPStore( # type: ignore[call-arg]
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
torch.distributed.DistNetworkError: The server socket has failed to listen on any local network address. port: 29500, useIpv6: false, code: -98, name: EADDRINUSE, message: address already in use
|
|
E0903 19:04:19.236000 1488504 /gpfs/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 2) local_rank: 0 (pid: 1488524) of binary: /home/nws8519/.conda/envs/olmo/bin/python3.11
|
|
E0903 19:04:19.236000 2554912 /gpfs/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/api.py:874] failed (exitcode: 2) local_rank: 0 (pid: 2554950) of binary: /home/nws8519/.conda/envs/olmo/bin/python3.11
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/torchrun", line 8, in <module>
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
|
Traceback (most recent call last):
|
|
File "/home/nws8519/.conda/envs/olmo/bin/torchrun", line 8, in <module>
|
|
return f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in main
|
|
sys.exit(main())
|
|
^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper
|
|
return f(*args, **kwargs)
|
|
^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 892, in main
|
|
run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
run(args)
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/run.py", line 883, in run
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
|
|
raise ChildFailedError(
|
|
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
|
|
============================================================
|
|
nnodes FAILED
|
|
------------------------------------------------------------
|
|
Failures:
|
|
<NO_OTHER_FAILURES>
|
|
------------------------------------------------------------
|
|
Root Cause (first observed failure):
|
|
[0]:
|
|
time : 2025-09-03_19:04:19
|
|
host : qgpu2013
|
|
rank : 0 (local_rank: 0)
|
|
exitcode : 2 (pid: 1488524)
|
|
error_file: <N/A>
|
|
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
|
|
============================================================
|
|
elastic_launch(
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 139, in __call__
|
|
return launch_agent(self._config, self._entrypoint, list(args))
|
|
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
|
File "/home/nws8519/.conda/envs/olmo/lib/python3.11/site-packages/torch/distributed/launcher/api.py", line 270, in launch_agent
|
|
raise ChildFailedError(
|
|
torch.distributed.elastic.multiprocessing.errors.ChildFailedError:
|
|
============================================================
|
|
nnodes FAILED
|
|
------------------------------------------------------------
|
|
Failures:
|
|
<NO_OTHER_FAILURES>
|
|
------------------------------------------------------------
|
|
Root Cause (first observed failure):
|
|
[0]:
|
|
time : 2025-09-03_19:04:19
|
|
host : qgpu2014
|
|
rank : 0 (local_rank: 0)
|
|
exitcode : 2 (pid: 2554950)
|
|
error_file: <N/A>
|
|
traceback : To enable traceback see: https://pytorch.org/docs/stable/elastic/errors.html
|
|
============================================================
|
|
srun: error: qgpu2013: tasks 0-1: Exited with exit code 1
|
|
srun: error: qgpu2014: tasks 2-3: Exited with exit code 1
|
|
unsupervised olmo categorization pau at Wed Sep 3 19:04:19 CDT 2025
|