Hanging with PETSc/Hypre/MPICH-5.0/HIP

I ran into the hanging when doing petsc `make check`. It ran fine with OpenMPI. I configured Hypre with or without Umpire, and got difference stack traces in hanging, which made me think this is a Hypre problem. So I report it here: To reproduce, ```shell $ cd $HOME $ git clone https://gitlab.com/petsc/petsc.git $ cd petsc $ export PETSC_DIR=$HOME/petsc $ export PETSC_ARCH=arch-kokkos-dbg ``` Then configure petsc with this python script ```python #!/usr/bin/env python3 import os if __name__ == '__main__': import os import sys sys.path.insert(0, os.path.abspath('config')) import configure configure_options = [ 'PETSC_ARCH=arch-kokkos-dbg', '--with-cc=gcc', '--with-cxx=g++', '--with-fc=0', '--with-hipc=/opt/rocm-5.4.3/bin/hipcc', '--with-hip-dir=/opt/rocm-5.4.3', 'COPTFLAGS=-g -O0', 'FOPTFLAGS=-g -O0', 'CXXOPTFLAGS=-g -O0', 'HIPOPTFLAGS=-g -O0', '--with-cuda=0', '--with-strict-petscerrorcode', '--download-hypre', '--download-umpire', '--download-mpich', ] configure.petsc_configure(configure_options) ``` After that, do `make` to finish the build. To run the test: ```shell $ cd src/snes/tutorials $ make ex19 ~/petsc/arch-kokkos-dbg/bin/mpiexec -n 2 ./ex19 -petsc_ci -dm_vec_type hip -dm_mat_type aijhipsparse -da_refine 3 -snes_monitor_short -ksp_norm_type unpreconditioned -pc_type hypre lid velocity = 0.0016, prandtl # = 1., grashof # = 1. 0 SNES Function norm 0.0406612 <hang> ``` Here are the stack traces of the two MPI processes ```shell (gdb) bt #0 0x00007f6bcfa7cf12 in printf_positional (s=s@entry=0x7fffd62c0610, format=format@entry=0x7f6be755af7f "%s(%d)", readonly_format=readonly_format@entry=0, ap=ap@entry=0x7fffd62c0790, ap_savep=ap_savep@entry=0x7fffd62c01a8, done=done@entry=0, nspecs_done=0, lead_str_end=0x7f6be755af7f "%s(%d)", work_buffer=0x7fffd62c01d0 "", save_errno=22, grouping=0x0, thousands_sep=0x7f6bcfbe244f "", mode_flags=0) at ./stdio-common/vfprintf-internal.c:1717 #1 0x00007f6bcfa7f336 in __vfprintf_internal (s=s@entry=0x7fffd62c0610, format=format@entry=0x7f6be755af7f "%s(%d)", ap=ap@entry=0x7fffd62c0790, mode_flags=mode_flags@entry=0) at ./stdio-common/vfprintf-internal.c:1602 #2 0x00007f6bcfa9249a in __vsnprintf_internal (string=0x7f6be96dba24 <ErrorRing+30276> "", maxlen=<optimized out>, format=0x7f6be755af7f "%s(%d)", args=args@entry=0x7fffd62c0790, mode_flags=mode_flags@entry=0) at ./libio/vsnprintf.c:114 #3 0x00007f6bcfa6a856 in __GI___snprintf (s=<optimized out>, maxlen=<optimized out>, format=<optimized out>) at ./stdio-common/snprintf.c:31 #4 0x00007f6bd0734895 in MPIR_Err_create_code_valist (lastcode=583288851, fatal=0, fcname=0x7f6be7498810 <__func__.2> "internal_Testall", line=129, error_class=19, generic_msg=0x7f6be749879f "**mpi_testall", specific_msg=0x7f6be74987ad "**mpi_testall %d %p %p %p", Argp=0x7fffd62c0a60) at src/mpi/errhan/errutil.c:1046 #5 0x00007f6bd073414b in MPIR_Err_create_code (lastcode=583288851, fatal=0, fcname=0x7f6be7498810 <__func__.2> "internal_Testall", line=129, error_class=15, generic_msg=0x7f6be749879f "**mpi_testall", specific_msg=0x7f6be74987ad "**mpi_testall %d %p %p %p") at src/mpi/errhan/errutil.c:871 #6 0x00007f6bd01c5392 in internal_Testall (count=5, array_of_requests=0x5641071be080, flag=0x7fffd62c0e30, array_of_statuses=0x5641072d2090) at src/binding/c/request/testall.c:129 #7 0x00007f6bd01c53fb in PMPI_Testall (count=5, array_of_requests=0x5641071be080, flag=0x7fffd62c0e30, array_of_statuses=0x5641072d2090) at src/binding/c/request/testall.c:166 #8 0x00007f6bc9965987 in hypre_MPI_Testall (count=5, array_of_requests=0x5641071be080, flag=0x7fffd62c0ea4, array_of_statuses=0x5641072d2090) at mpistubs.c:1455 #9 0x00007f6bc995df11 in hypre_DataExchangeList (num_contacts=5, contact_proc_list=0x7f67e5cef010, contact_send_buf=0x7f67e35bc010, contact_send_buf_starts=0x7f67e4fde010, contact_obj_size=4, response_obj_size=4, response_obj=0x7fffd62c11a0, max_response_size=6, rnum=1, comm=-2080374778, p_response_recv_buf=0x7fffd62c1148, p_response_recv_buf_starts=0x7fffd62c1150) at exchange_data.c:424 #10 0x00007f6bc93f8bec in hypre_ParCSRCommPkgCreateApart_core (comm=-2080374778, col_map_off_d=0x5641072caeb0, first_col_diag=0, num_cols_off_d=12, global_num_cols=838, p_num_recvs=0x7fffd62c12ac, p_recv_procs=0x7fffd62c12c0, p_recv_vec_starts=0x7fffd62c12c8, p_num_sends=0x7fffd62c12a8, p_send_procs=0x7fffd62c12b0, p_send_map_starts=0x7fffd62c12b8, p_send_map_elements=0x7fffd62c12d0, apart=0x5641072cbe50) at new_commpkg.c:263 #11 0x00007f6bc93f95f7 in hypre_ParCSRCommPkgCreateApart (comm=-2080374778, col_map_off_d=0x5641072caeb0, first_col_diag=0, num_cols_off_d=12, global_num_cols=838, apart=0x5641072cbe50, comm_pkg=0x5641071c41d0) at new_commpkg.c:548 #12 0x00007f6bc9404bc6 in hypre_MatvecCommPkgCreate (A=0x5641072cb410) at par_csr_communication.c:1214 #13 0x00007f6bc937f4c7 in hypre_BoomerAMGSetup (amg_vdata=0x564104f2ba70, A=0x5641055ed5c0, f=0x564104d346d0, u=0x5641057c3a30) at par_amg_setup.c:3107 #14 0x00007f6bc91f00c7 in HYPRE_BoomerAMGSetup (solver=0x564104f2ba70, A=0x5641055ed5c0, b=0x564104d346d0, x=0x5641057c3a30) at HYPRE_parcsr_amg.c:53 #15 0x00007f6bed52e9f7 in PCSetUp_HYPRE (pc=0x564104c8ae00) at /scratch/jczhang/petsc/src/ksp/pc/impls/hypre/hypre.c:557 #16 0x00007f6bed7ffeb3 in PCSetUp (pc=0x564104c8ae00) at /scratch/jczhang/petsc/src/ksp/pc/interface/precon.c:1120 #17 0x00007f6becef6a99 in KSPSetUp (ksp=0x564104d58290) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:429 #18 0x00007f6becf01025 in KSPSolve_Private (ksp=0x564104d58290, b=0x564104d8b060, x=0x564104d7e5b0) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:840 #19 0x00007f6becf0682f in KSPSolve (ksp=0x564104d58290, b=0x564104d8b060, x=0x564104d7e5b0) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:1093 #20 0x00007f6bed8a053e in SNESSolve_NEWTONLS (snes=0x564104b1a740) at /scratch/jczhang/petsc/src/snes/impls/ls/ls.c:220 #21 0x00007f6beda4bbca in SNESSolve (snes=0x564104b1a740, b=0x0, x=0x564104e78e60) at /scratch/jczhang/petsc/src/snes/interface/snes.c:4900 #22 0x00005640f77aadfe in main (argc=13, argv=0x7fffd62c4b18) at ex19.c:152 ``` ```shell (gdb) bt #0 MPIDI_OFI_has_cq_buffered (vci=0) at ./src/mpid/ch4/netmod/include/../ofi/ofi_impl.h:645 #1 0x00007fe286c42653 in MPIDI_NM_progress (vci=0, made_progress=0x7fffd4ca8d64) at ./src/mpid/ch4/netmod/include/../ofi/ofi_progress.h:88 #2 0x00007fe286c45a19 in MPIDI_progress_test (state=0x7fffd4ca8eb0) at ./src/mpid/ch4/src/ch4_progress.h:172 #3 0x00007fe286c46378 in MPID_Progress_test (state=0x7fffd4ca8eb0) at ./src/mpid/ch4/src/ch4_progress.h:271 #4 0x00007fe286c483eb in MPIR_Test_state (request_ptr=0x558676725950, flag=0x7fffd4ca9000, status=0x7fffd4ca9190, state=0x7fffd4ca8eb0) at src/mpi/request/request_impl.c:203 #5 0x00007fe286c47912 in MPID_Test (request_ptr=0x558676725950, flag=0x7fffd4ca9000, status=0x7fffd4ca9190) at ./src/mpid/ch4/src/ch4_wait.h:59 #6 0x00007fe286c484d8 in MPIR_Test (request_ptr=0x558676725950, flag=0x7fffd4ca9000, status=0x7fffd4ca9190) at src/mpi/request/request_impl.c:223 #7 0x00007fe286651681 in internal_Test (request=0x7fffd4ca9088, flag=0x7fffd4ca9000, status=0x7fffd4ca9190) at src/binding/c/request/test.c:86 #8 0x00007fe286651769 in PMPI_Test (request=0x7fffd4ca9088, flag=0x7fffd4ca9000, status=0x7fffd4ca9190) at src/binding/c/request/test.c:142 #9 0x00007fe27fdf5922 in hypre_MPI_Test (request=0x7fffd4ca9088, flag=0x7fffd4ca9070, status=0x7fffd4ca9190) at mpistubs.c:1438 #10 0x00007fe27fdee046 in hypre_DataExchangeList (num_contacts=1, contact_proc_list=0x55867770a5a0, contact_send_buf=0x55867770a560, contact_send_buf_starts=0x558677707260, contact_obj_size=4, response_obj_size=4, response_obj=0x7fffd4ca9370, max_response_size=6, rnum=1, comm=-2080374780, p_response_recv_buf=0x7fffd4ca9318, p_response_recv_buf_starts=0x7fffd4ca9320) at exchange_data.c:454 #11 0x00007fe27f888bec in hypre_ParCSRCommPkgCreateApart_core (comm=-2080374780, col_map_off_d=0x558676bd5490, first_col_diag=439, num_cols_off_d=76, global_num_cols=838, p_num_recvs=0x7fffd4ca947c, p_recv_procs=0x7fffd4ca9490, p_recv_vec_starts=0x7fffd4ca9498, p_num_sends=0x7fffd4ca9478, p_send_procs=0x7fffd4ca9480, p_send_map_starts=0x7fffd4ca9488, p_send_map_elements=0x7fffd4ca94a0, apart=0x558677707de0) at new_commpkg.c:263 #12 0x00007fe27f8895f7 in hypre_ParCSRCommPkgCreateApart (comm=-2080374780, col_map_off_d=0x558676bd5490, first_col_diag=439, num_cols_off_d=76, global_num_cols=838, apart=0x558677707de0, comm_pkg=0x5586777107d0) at new_commpkg.c:548 #13 0x00007fe27f894bc6 in hypre_MatvecCommPkgCreate (A=0x558677712f60) at par_csr_communication.c:1214 #14 0x00007fe27f80f4c7 in hypre_BoomerAMGSetup (amg_vdata=0x5586754bed90, A=0x558675a5dfb0, f=0x558675d0c8a0, u=0x558675b93590) at par_amg_setup.c:3107 #15 0x00007fe27f6800c7 in HYPRE_BoomerAMGSetup (solver=0x5586754bed90, A=0x558675a5dfb0, b=0x558675d0c8a0, x=0x558675b93590) at HYPRE_parcsr_amg.c:53 #16 0x00007fe2a39be9f7 in PCSetUp_HYPRE (pc=0x558675587f80) at /scratch/jczhang/petsc/src/ksp/pc/impls/hypre/hypre.c:557 #17 0x00007fe2a3c8feb3 in PCSetUp (pc=0x558675587f80) at /scratch/jczhang/petsc/src/ksp/pc/interface/precon.c:1120 #18 0x00007fe2a3386a99 in KSPSetUp (ksp=0x5586755dbef0) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:429 #19 0x00007fe2a3391025 in KSPSolve_Private (ksp=0x5586755dbef0, b=0x5586751c4630, x=0x55867556f8a0) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:840 #20 0x00007fe2a339682f in KSPSolve (ksp=0x5586755dbef0, b=0x5586751c4630, x=0x55867556f8a0) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:1093 #21 0x00007fe2a3d3053e in SNESSolve_NEWTONLS (snes=0x55867547a860) at /scratch/jczhang/petsc/src/snes/impls/ls/ls.c:220 #22 0x00007fe2a3edbbca in SNESSolve (snes=0x55867547a860, b=0x0, x=0x5586754c6370) at /scratch/jczhang/petsc/src/snes/interface/snes.c:4900 #23 0x000055865cca3dfe in main (argc=13, argv=0x7fffd4cacce8) at ex19.c:152 ``` If I configured Hypre without Umpire, the stack traces were similar on the two processes: ```shell #11 0x00007f109cf3c577 in hipStreamSynchronize () from /opt/rocm-5.4.3/lib/libamdhip64.so.5 #12 0x00007f109f088ff3 in thrust::hip_rocprim::synchronize_stream_optional<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base> > (policy=...) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/util.h:116 #13 0x00007f109f08737d in thrust::hip_rocprim::synchronize_optional<thrust::hip_rocprim::execution_policy<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base> > > (policy=...) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/util.h:144 #14 0x00007f109f09a9af in thrust::hip_rocprim::parallel_for<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, thrust::hip_rocprim::__transform::unary_transform_f<thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::__transform::no_stencil_tag, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate>, long>(thrust::hip_rocprim::execution_policy<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base> >&, thrust::hip_rocprim::__transform::unary_transform_f<thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::__transform::no_stencil_tag, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate>, long)::workaround::par(thrust::hip_rocprim::execution_policy<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base> >&, thrust::hip_rocprim::__transform::unary_transform_f<thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::__transform::no_stencil_tag, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate>, long) (policy=..., f=..., count=24) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/parallel_for.h:150 #15 0x00007f109f0806c2 in thrust::hip_rocprim::parallel_for<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, thrust::hip_rocprim::__transform::unary_transform_f<thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::__transform::no_stencil_tag, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate>, long> (policy=..., f=..., count=24) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/parallel_for.h:166 #16 thrust::hip_rocprim::__transform::unary<thrust::hip_rocprim::execution_policy<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base> >, thrust::permutation_iterator<int*, int*>, long, int*, thrust::hip_rocprim::__transform::no_stencil_tag, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate> (policy=..., items=..., result=0x7f0ce0f6d000, num_items=24) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/transform.h:221 #17 thrust::hip_rocprim::transform_if<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::__transform::no_stencil_tag, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate> (policy=..., first=..., last=..., result=0x7f0ce0f6d000) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/transform.h:291 #18 thrust::hip_rocprim::transform_if<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::identity, thrust::hip_rocprim::__transform::always_true_predicate> (policy=..., first=..., last=..., result=0x7f0ce0f6d000) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/transform.h:305 #19 thrust::hip_rocprim::transform<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, thrust::permutation_iterator<int*, int*>, int*, thrust::hip_rocprim::identity> (policy=..., first=..., last=..., result=0x7f0ce0f6d000) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/transform.h:318 #20 thrust::hip_rocprim::gather<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, int*, int*, int*> (policy=..., map_first=0x7f0ce0f6b000, map_last=0x7f0ce0f6b060, items=0x7f0ce0f65000, result=0x7f0ce0f6d000) at /opt/rocm-5.4.3/include/thrust/system/hip/detail/gather.h:46 #21 thrust::gather<thrust::detail::execute_with_allocator<hypre_device_allocator&, thrust::hip_rocprim::execute_on_stream_base>, int*, int*, int*> (exec=..., map_first=0x7f0ce0f6b000, map_last=0x7f0ce0f6b060, input_first=0x7f0ce0f65000, result=0x7f0ce0f6d000) at /opt/rocm-5.4.3/include/thrust/detail/gather.inl:42 #22 0x00007f109f2c8f9f in hypre_BoomerAMGCoarsenPMISDevice (S=0x5653f6621e20, A=0x5653f5f458f0, CF_init=0, debug_flag=0, CF_marker_ptr=0x5653f60c3510) at par_coarsen_device.c:211 #23 0x00007f109f17d6de in hypre_BoomerAMGCoarsenPMIS (S=0x5653f6621e20, A=0x5653f5f458f0, CF_init=0, debug_flag=0, CF_marker_ptr=0x5653f60c3510) at par_coarsen.c:2829 #24 0x00007f109f2b3720 in hypre_BoomerAMGSetup (amg_vdata=0x5653f5b62140, A=0x5653f5f458f0, f=0x5653f5a1f270, u=0x5653f6075f50) at par_amg_setup.c:1239 #25 0x00007f109f129ff7 in HYPRE_BoomerAMGSetup (solver=0x5653f5b62140, A=0x5653f5f458f0, b=0x5653f5a1f270, x=0x5653f6075f50) at HYPRE_parcsr_amg.c:53 #26 0x00007f10c3466843 in PCSetUp_HYPRE (pc=0x5653f5ac8b30) at /scratch/jczhang/petsc/src/ksp/pc/impls/hypre/hypre.c:557 #27 0x00007f10c3737cff in PCSetUp (pc=0x5653f5ac8b30) at /scratch/jczhang/petsc/src/ksp/pc/interface/precon.c:1120 #28 0x00007f10c2e2ed4f in KSPSetUp (ksp=0x5653f5504890) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:429 #29 0x00007f10c2e392db in KSPSolve_Private (ksp=0x5653f5504890, b=0x5653f5966f10, x=0x5653f5b79e60) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:840 #30 0x00007f10c2e3eae5 in KSPSolve (ksp=0x5653f5504890, b=0x5653f5966f10, x=0x5653f5b79e60) at /scratch/jczhang/petsc/src/ksp/ksp/interface/itfunc.c:1093 #31 0x00007f10c37d838a in SNESSolve_NEWTONLS (snes=0x5653f5976110) at /scratch/jczhang/petsc/src/snes/impls/ls/ls.c:220 #32 0x00007f10c3983a16 in SNESSolve (snes=0x5653f5976110, b=0x0, x=0x5653f5b69720) at /scratch/jczhang/petsc/src/snes/interface/snes.c:4900 #33 0x00005653e8becdfe in main (argc=13, argv=0x7ffddaf3a5a8) at ex19.c:152 ```