dpgen2 cannot submit: urllib3.exceptions.MaxRetryError
Hello developers,
I encountered the following issue while running dpgen2. The command I used was: `dpgen2 submit input.json.` However, the following problem occurred right at startup:
```bash
(base) tieyuan@polyucmp-GPU:03.distillation$ dpgen2 submit input.json
CP2KDATA| Retrying (Retry(total=4, connect=None, read=None, redirect=None, status=None)) after connection broken by 'BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n')': /my-bucket?location=
CP2KDATA| Retrying (Retry(total=3, connect=None, read=None, redirect=None, status=None)) after connection broken by 'BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n')': /my-bucket?location=
CP2KDATA| Retrying (Retry(total=2, connect=None, read=None, redirect=None, status=None)) after connection broken by 'BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n')': /my-bucket?location=
CP2KDATA| Retrying (Retry(total=1, connect=None, read=None, redirect=None, status=None)) after connection broken by 'BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n')': /my-bucket?location=
CP2KDATA| Retrying (Retry(total=0, connect=None, read=None, redirect=None, status=None)) after connection broken by 'BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n')': /my-bucket?location=
urllib3.exceptions.ProtocolError: ('Connection aborted.', BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/bin/dpgen2", line 8, in <module>
sys.exit(main())
^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dpgen2/entrypoint/main.py", line 335, in main
submit_concurrent_learning(
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dpgen2/entrypoint/submit.py", line 754, in submit_concurrent_learning
dpgen_step = workflow_concurrent_learning(wf_config)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dpgen2/entrypoint/submit.py", line 540, in workflow_concurrent_learning
valid_data = upload_artifact_and_print_uri(valid_data, "valid_data")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dpgen2/utils/artifact_uri.py", line 18, in upload_artifact_and_print_uri
art = upload_artifact(files)
^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dflow/utils.py", line 276, in upload_artifact
key = upload_s3(path=tmpdir + ".tgz", **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dflow/utils.py", line 416, in upload_s3
client.upload(key=key, path=path)
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/dflow/utils.py", line 866, in upload
self.client.fput_object(bucket_name=self.bucket_name,
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 1053, in fput_object
return self.put_object(
^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 2008, in put_object
raise exc
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 1954, in put_object
return self._put_object(
^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 1783, in _put_object
response = self._execute(
^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 440, in _execute
region = self._get_region(bucket_name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 497, in _get_region
response = self._url_open(
^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/minio/api.py", line 305, in _url_open
response = self._http.urlopen(
^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/urllib3/poolmanager.py", line 443, in urlopen
response = conn.urlopen(method, u.request_uri, **kw)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/urllib3/connectionpool.py", line 871, in urlopen
return self.urlopen(
^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/urllib3/connectionpool.py", line 871, in urlopen
return self.urlopen(
^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/urllib3/connectionpool.py", line 871, in urlopen
return self.urlopen(
^^^^^^^^^^^^^
[Previous line repeated 2 more times]
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/urllib3/connectionpool.py", line 841, in urlopen
retries = retries.increment(
^^^^^^^^^^^^^^^^^^
File "/mnt/tieyuan/code/deepmd-kit-3.1.0/lib/python3.12/site-packages/urllib3/util/retry.py", line 519, in increment
raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
urllib3.exceptions.MaxRetryError: HTTPConnectionPool(host='127.0.0.1', port=9000): Max retries exceeded with url: /my-bucket?location= (Caused by ProtocolError('Connection aborted.', BadStatusLine('ÿ\x00\x00\x00\x00\x00\x00\x00\x01\x7f0\x000.1:9000\r\n')))
```
Here is my input.json:
```java
{
"name": "pervoskite-distill",
"default_step_config": {
"template_config": {
"envs": {},
"_comment": "all"
}
},
"step_configs": {
"run_train_config": {
"executor": {
"type": "dispatcher",
"retry_on_submission_error": 10,
"machine_dict": {
"batch_type": "Slurm",
"context_type": "LocalContext",
"local_root": "./",
"remote_root": "./"
},
"resources_dict": {
"number_node": 1,
"cpu_per_node": 16,
"gpu_per_node": 1,
"queue_name": "4GPUs",
"group_size": 1,
"module_list": [
"cuda/12.8"
],
"custom_flags": [
],
"source_list": [
"/mnt/tieyuan/code/deepmd-kit-3.1.0/bin/activate"
],
"envs": {
"OMP_NUM_THREADS": "8"
}
},
"task_dict": {
"_command": "lmp -i input.lammps",
"task_work_path": "./",
"outlog": "log",
"errlog": "err",
"forward_files": [
"script"
],
"backward_files": []
}
},
"_comment": "all"
},
"run_explore_config": {
"continue_on_success_ratio": 0.8,
"executor": {
"type": "dispatcher",
"retry_on_submission_error": 10,
"image_pull_policy": "IfNotPresent",
"machine_dict": {
"batch_type": "Slurm",
"context_type": "LocalContext",
"local_root": "./",
"remote_root": "./"
},
"resources_dict": {
"number_node": 1,
"cpu_per_node": 16,
"gpu_per_node": 1,
"queue_name": "4GPUs",
"group_size": 10,
"module_list": [
"cuda/12.8"
],
"custom_flags": [
],
"source_list": [
"/mnt/tieyuan/code/deepmd-kit-3.1.0/bin/activate"
],
"envs": {
"OMP_NUM_THREADS": "8"
}
},
"task_dict": {
"_command": "lmp -i input.lammps",
"task_work_path": "./",
"outlog": "log",
"errlog": "err",
"forward_files": [
"script"
],
"backward_files": []
}
},
"_comment": "all"
},
"run_fp_config": {
"continue_on_success_ratio": 0.01,
"executor": {
"type": "dispatcher",
"retry_on_submission_error": 10,
"image_pull_policy": "IfNotPresent",
"machine_dict": {
"batch_type": "Slurm",
"context_type": "LocalContext",
"local_root": "./",
"remote_root": "./"
},
"resources_dict": {
"number_node": 1,
"cpu_per_node": 4,
"gpu_per_node": 1,
"queue_name": "4GPUs",
"group_size": 100,
"module_list": [
"cuda/12.8"
],
"custom_flags": [
],
"envs": {
"OMP_NUM_THREADS": "8"
}
},
"task_dict": {
"_command": "lmp -i input.lammps",
"task_work_path": "./",
"outlog": "log",
"errlog": "err",
"forward_files": [
"script"
],
"backward_files": []
}
},
"template_slice_config": {
"group_size": 10,
"pool_size": 1
}
},
"_comment": "all"
},
"upload_python_packages": [
"/mnt/tieyuan/code/deepmd-kit-3.1.0/bin/dpgen2",
"/mnt/tieyuan/code/deepmd-kit-3.1.0/bin/dpdata"
],
"inputs": {
"type_map": [
"Br",
"C",
"H",
"I",
"N",
"Pb"
],
"mixed_type": true,
"mass_map": [
79.904,
12.011,
1.008,
126.90447,
14.007,
207.2
],
"init_data_prefix": null,
"init_data_sys": [
"train_predict/Br0C12H71I36N13Pb12",
"train_predict/Br0C32H168I96N56Pb32",
"train_predict/Br0C32H179I96N45Pb32",
"train_predict/Br0C27H135I81N54Pb27",
"train_predict/Br69C27H135I13N54Pb27",
"train_predict/Br0C32H175I96N49Pb32",
"train_predict/Br0C16H94I48N18Pb16",
"train_predict/Br0C32H174I96N50Pb32",
"train_predict/Br0C32H163I96N61Pb32",
"train_predict/Br0C42H108I38N14Pb12",
"train_predict/Br0C120H208I56N24Pb16",
"train_predict/Br32C16H80I16N32Pb16",
"train_predict/Br0C16H87I48N25Pb16",
"train_predict/Br0C32H64I20N8Pb6",
"train_predict/Br0C32H177I96N47Pb32",
"train_predict/Br0C112H160I32N16Pb8",
"train_predict/Br14C27H135I66N54Pb27",
"train_predict/Br24C16H80I24N32Pb16",
"train_predict/Br0C48H128I16N8Pb4",
"train_predict/Br0C128H384I64N32Pb16",
"train_predict/Br0C8H43I24N13Pb8",
"train_predict/Br0C16H80I48N32Pb20",
"train_predict/Br0C16H83I48N29Pb16",
"train_predict/Br0C16H81I48N31Pb16",
"train_predict/Br0C12H66I36N18Pb12",
"train_predict/Br16C16H80I32N32Pb16",
"train_predict/Br0C16H89I48N23Pb16",
"train_predict/Br0C27H149I81N40Pb27",
"train_predict/Br0C4H20I10N8Pb3",
"train_predict/Br0C16H48I8N4Pb2",
"train_predict/Br0C32H190I96N34Pb32",
"train_predict/Br0C38H84I26N10Pb8",
"train_predict/Br0C2H0I0N0Pb0",
"train_predict/Br0C32H187I96N37Pb32",
"train_predict/Br0C0H0I0N2Pb0",
"train_predict/Br67C27H135I13N54Pb27",
"train_predict/Br0C32H185I96N39Pb32",
"train_predict/Br0C0H1I0N1Pb0",
"train_predict/Br0C16H88I48N24Pb16",
"train_predict/Br0C16H82I48N30Pb16",
"train_predict/Br0C32H186I96N38Pb32",
"train_predict/Br54C27H135I27N54Pb27",
"train_predict/Br0C4H20I12N8Pb4",
"train_predict/Br0C32H161I96N63Pb32",
"train_predict/Br0C32H170I96N54Pb32",
"train_predict/Br0C28H40I8N4Pb2",
"train_predict/Br0C2H11I6N3Pb2",
"train_predict/Br69C27H135I14N54Pb27",
"train_predict/Br48C16H80I0N32Pb16",
"train_predict/Br0C32H182I96N42Pb32",
"train_predict/Br0C128H256I80N32Pb24",
"train_predict/Br0C24H132I72N36Pb24",
"train_predict/Br0C68H120I28N12Pb8",
"train_predict/Br0C4H21I12N7Pb4",
"train_predict/Br0C32H160I96N64Pb32",
"train_predict/Br0C76H168I52N20Pb16",
"train_predict/Br0C20H100I60N40Pb16",
"train_predict/Br0C34H60I14N6Pb4",
"train_predict/Br0C8H47I24N9Pb8",
"train_predict/Br0C16H85I48N27Pb16",
"train_predict/Br0C16H91I48N21Pb16",
"train_predict/Br15C27H135I68N54Pb27",
"train_predict/Br0C32H165I96N59Pb32",
"train_predict/Br0C4H23I12N5Pb4",
"train_predict/Br0C32H176I96N48Pb32",
"train_predict/Br0C16H90I48N22Pb16",
"train_predict/Br0C1H6I3N1Pb1",
"train_predict/Br0C1H1I0N0Pb0",
"train_predict/Br0C0H2I0N0Pb0",
"train_predict/Br0C32H48I8N4Pb2",
"train_predict/Br0C12H61I36N23Pb12",
"train_predict/Br0C8H40I24N16Pb8",
"train_predict/Br0C1H0I0N1Pb0",
"train_predict/Br40C27H135I41N54Pb27",
"train_predict/Br0C34H76I26N10Pb8",
"train_predict/Br0C32H164I96N60Pb32",
"train_predict/Br0C40H96I32N12Pb10",
"train_predict/Br0C32H173I96N51Pb32",
"train_predict/Br0C32H167I96N57Pb32",
"train_predict/Br0C72H144I40N16Pb12",
"train_predict/Br14C27H135I68N54Pb27",
"train_predict/Br0C16H93I48N19Pb16",
"train_predict/Br0C36H180I108N72Pb36",
"train_predict/Br0C2H10I6N4Pb2",
"train_predict/Br0C32H180I96N44Pb32",
"train_predict/Br0C64H90I16N8Pb4",
"train_predict/Br0C136H240I56N24Pb16",
"train_predict/Br0C8H48I24N8Pb8",
"train_predict/Br0C32H178I96N46Pb32",
"train_predict/Br0C128H192I32N16Pb8",
"train_predict/Br0C30H52I14N6Pb4",
"train_predict/Br0C16H80I48N32Pb16",
"train_predict/Br0C32H166I96N58Pb32",
"train_predict/Br0C60H104I28N12Pb8",
"train_predict/Br27C27H135I54N54Pb27",
"train_predict/Br0C16H86I48N26Pb16",
"train_predict/Br0C144H288I80N32Pb24",
"train_predict/Br0C32H188I96N36Pb32",
"train_predict/Br0C16H84I48N28Pb16",
"train_predict/Br0C32H181I96N43Pb32",
"train_predict/Br0C32H192I96N32Pb32",
"train_predict/Br0C32H189I96N35Pb32",
"train_predict/Br0C32H184I96N40Pb32",
"train_predict/Br0C4H24I12N4Pb4",
"train_predict/Br0C32H169I96N55Pb32",
"train_predict/Br0C32H96I16N8Pb4",
"train_predict/Br0C4H22I12N6Pb4",
"train_predict/Br0C8H46I24N10Pb8",
"train_predict/Br67C27H135I12N54Pb27",
"train_predict/Br0C64H128I40N16Pb12",
"train_predict/Br0C8H41I24N15Pb8",
"train_predict/Br0C8H44I24N12Pb8",
"train_predict/Br0C16H95I48N17Pb16",
"train_predict/Br0C16H96I48N16Pb16",
"train_predict/Br0C1H5I3N2Pb1",
"train_predict/Br0C12H60I44N24Pb16",
"train_predict/Br81C27H135I0N54Pb27",
"train_predict/Br0C32H172I96N52Pb32",
"train_predict/Br0C32H183I96N41Pb32",
"train_predict/Br0C36H72I20N8Pb6",
"train_predict/Br0C64H192I32N16Pb8",
"train_predict/Br13C27H135I66N54Pb27",
"train_predict/Br0C8H45I24N11Pb8",
"train_predict/Br0C8H42I24N14Pb8",
"train_predict/Br0C24H138I72N30Pb24",
"train_predict/Br0C16H92I48N20Pb16",
"train_predict/Br0C38H100I38N14Pb12",
"train_predict/Br0C32H191I96N33Pb32",
"train_predict/Br0C16H80I40N32Pb12",
"train_predict/Br0C64H96I16N8Pb4",
"train_predict/Br0C12H60I36N24Pb12",
"train_predict/Br0C56H80I16N8Pb4",
"train_predict/Br0C32H171I96N53Pb32",
"train_predict/Br0C36H88I32N12Pb10"
],
"valid_data_sys": [
"test_predict/C27N54H135Pb27Br42I39",
"test_predict/C16N32H80Pb16Br21I28",
"test_predict/C64N128H320Pb64Br96I96"
],
"_comment": "all"
},
"train": {
"type": "dp",
"numb_models": 4,
"config": {
"init_model_policy": "yes",
"init_model_old_ratio": 0.9,
"init_model_numb_steps": 800000,
"init_model_start_lr": 0.0001,
"init_model_start_pref_e": 1.0,
"init_model_start_pref_f": 100,
"init_model_start_pref_v": 1.0,
"impl": "tensorflow",
"_comment": "all"
},
"template_script": "train.json",
"_comment": "all"
},
"explore": {
"type": "lmp",
"config": {
"command": "lmp -var restart 0"
},
"convergence": {
"type": "adaptive-lower",
"conv_tolerance": 0.005,
"_numb_candi_f": 3000,
"rate_candi_f": 0.5,
"level_f_hi": 3.0,
"n_checked_steps": 2,
"_command": "all"
},
"max_numb_iter": 16,
"fatal_at_max": false,
"configuration_prefix": null,
"configurations": [
{
"type": "file",
"files": [
"init"
],
"fmt": "deepmd/npy/mixed"
}
],
"stages": [
[
{
"type": "lmp-template",
"lmp": "template_npt.lammps",
"trj_freq": 100,
"revisions": {
"V_NSTEPS": [
10000
],
"V_TEMP": [
300,
500,
800,
1200
],
"V_PRESS": [
0,
10000,
20000,
40000,
80000
],
"V_DUMPFREQ": [
200
]
},
"sys_idx": [
0
],
"n_sample": 2
}
]
],
"_comment": "all"
},
"fp": {
"type": "deepmd",
"task_max": 4000,
"run_config": {
"teacher_model_path": "teacher_model.pt"
},
"inputs_config": {},
"_comment": "all"
}
}
```
Could you please help me understand what might be causing this issue?
Thank you!
关闭于 2025-09-02 5 条评论