Topic: OOFEM only using one single thread

Today I tried to solve a heavy problem on our station, but the analysis is very slow even compared to my laptop.

I noticed that OOFEM only uses one thread leaving other CPU cores free.

To my knowledge the matrix solvers can do multithreading even if the analysis is not parallel,
am I missing something that the analysis is running with only one thread?

Re: OOFEM only using one single thread

You need to compile it with OPENMP, so just specify USE_OPENMP ON in the cmake. Then you can use  -t #nthreads

Re: OOFEM only using one single thread

Thanks for the tip. I get the following error:

====================[ Build | oofem | Release_OMP ]=============================
[  5%] Built target fm
[ 12%] Built target tm
[ 47%] Built target core
[ 98%] Built target sm
[ 98%] Built target sm
[100%] Linking CXX shared library liboofem.dylib
Undefined symbols for architecture x86_64:
  "___kmpc_barrier", referenced from:
      oofem::OctreeSpatialLocalizer::init(bool) in octreelocalizer.C.o
  "___kmpc_critical", referenced from:
      _.omp_outlined. in engngm.C.o
      _.omp_outlined..35 in engngm.C.o
      _.omp_outlined..36 in engngm.C.o
      _.omp_outlined..37 in engngm.C.o
      _.omp_outlined..62 in engngm.C.o
      _.omp_outlined..66 in engngm.C.o
      oofem::EngngModel::assembleExtrapolatedForces(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::Domain*) in engngm.C.o
      ...
  "___kmpc_end_critical", referenced from:
      _.omp_outlined. in engngm.C.o
      _.omp_outlined..35 in engngm.C.o
      _.omp_outlined..36 in engngm.C.o
      _.omp_outlined..37 in engngm.C.o
      _.omp_outlined..62 in engngm.C.o
      _.omp_outlined..66 in engngm.C.o
      oofem::EngngModel::assembleExtrapolatedForces(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::Domain*) in engngm.C.o
      ...
  "___kmpc_end_single", referenced from:
      oofem::OctreeSpatialLocalizer::init(bool) in octreelocalizer.C.o
  "___kmpc_for_static_fini", referenced from:
      _.omp_outlined. in engngm.C.o
      _.omp_outlined..34 in engngm.C.o
      _.omp_outlined..35 in engngm.C.o
      _.omp_outlined..36 in engngm.C.o
      _.omp_outlined..37 in engngm.C.o
      _.omp_outlined..62 in engngm.C.o
      _.omp_outlined..64 in engngm.C.o
      ...
  "___kmpc_for_static_init_4", referenced from:
      _.omp_outlined. in engngm.C.o
      _.omp_outlined..34 in engngm.C.o
      _.omp_outlined..35 in engngm.C.o
      _.omp_outlined..36 in engngm.C.o
      _.omp_outlined..37 in engngm.C.o
      _.omp_outlined..62 in engngm.C.o
      _.omp_outlined..66 in engngm.C.o
      ...
  "___kmpc_for_static_init_8", referenced from:
      _.omp_outlined..64 in engngm.C.o
      _.omp_outlined..67 in engngm.C.o
  "___kmpc_fork_call", referenced from:
      oofem::EngngModel::assembleVectorFromDofManagers(oofem::FloatArray&, oofem::TimeStep*, oofem::VectorAssembler const&, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::Domain*, oofem::FloatArray*) in engngm.C.o
      oofem::EngngModel::assembleVectorFromBC(oofem::FloatArray&, oofem::TimeStep*, oofem::VectorAssembler const&, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::Domain*, oofem::FloatArray*) in engngm.C.o
      oofem::EngngModel::assembleVectorFromElements(oofem::FloatArray&, oofem::TimeStep*, oofem::VectorAssembler const&, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::Domain*, oofem::FloatArray*) in engngm.C.o
      oofem::EngngModel::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::MatrixAssembler const&, oofem::UnknownNumberingScheme const&, oofem::Domain*) in engngm.C.o
      oofem::EngngModel::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::MatrixAssembler const&, oofem::UnknownNumberingScheme const&, oofem::UnknownNumberingScheme const&, oofem::Domain*) in engngm.C.o
      oofem::EngngModel::assembleVector(oofem::FloatArray&, oofem::TimeStep*, oofem::VectorAssembler const&, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::Domain*, oofem::FloatArray*) in engngm.C.o
      oofem::TransportGradientDirichlet::computeTangent(oofem::FloatMatrix&, oofem::TimeStep*) in transportgradientdirichlet.C.o
      ...
  "___kmpc_global_thread_num", referenced from:
      oofem::OctreeSpatialLocalizer::init(bool) in octreelocalizer.C.o
      oofem::EngngModel::assembleExtrapolatedForces(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::Domain*) in engngm.C.o
      oofem::EngngModel::assemblePrescribedExtrapolatedForces(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::Domain*) in engngm.C.o
  "___kmpc_single", referenced from:
      oofem::OctreeSpatialLocalizer::init(bool) in octreelocalizer.C.o
  "_omp_init_lock", referenced from:
      oofem::OctreeSpatialLocalizer::OctreeSpatialLocalizer(oofem::Domain*) in octreelocalizer.C.o
      oofem::EngngModel::assembleVectorFromBC(oofem::FloatArray&, oofem::TimeStep*, oofem::VectorAssembler const&, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::Domain*, oofem::FloatArray*) in engngm.C.o
      oofem::EngngModel::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::MatrixAssembler const&, oofem::UnknownNumberingScheme const&, oofem::Domain*) in engngm.C.o
      oofem::EngngModel::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::MatrixAssembler const&, oofem::UnknownNumberingScheme const&, oofem::UnknownNumberingScheme const&, oofem::Domain*) in engngm.C.o
      oofem::EngngModel::assembleVector(oofem::FloatArray&, oofem::TimeStep*, oofem::VectorAssembler const&, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::Domain*, oofem::FloatArray*) in engngm.C.o
  "_omp_set_lock", referenced from:
      oofem::WeakPeriodicBoundaryCondition::giveInternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::FloatArray*, void*) in weakperiodicbc.C.o
      oofem::WeakPeriodicBoundaryCondition::giveExternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, void*) in weakperiodicbc.C.o
      oofem::WeakPeriodicBoundaryCondition::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::CharType, oofem::UnknownNumberingScheme const&, oofem::UnknownNumberingScheme const&, double, void*) in weakperiodicbc.C.o
      oofem::PrescribedMean::giveInternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::FloatArray*, void*) in prescribedmean.C.o
      oofem::PrescribedMean::giveExternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, void*) in prescribedmean.C.o
      oofem::PrescribedMean::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::CharType, oofem::UnknownNumberingScheme const&, oofem::UnknownNumberingScheme const&, double, void*) in prescribedmean.C.o
      oofem::OctreeSpatialLocalizer::initElementIPDataStructure() in octreelocalizer.C.o
      ...
  "_omp_unset_lock", referenced from:
      oofem::WeakPeriodicBoundaryCondition::giveInternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::FloatArray*, void*) in weakperiodicbc.C.o
      oofem::WeakPeriodicBoundaryCondition::giveExternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, void*) in weakperiodicbc.C.o
      oofem::WeakPeriodicBoundaryCondition::assemble(oofem::SparseMtrx&, oofem::TimeStep*, oofem::CharType, oofem::UnknownNumberingScheme const&, oofem::UnknownNumberingScheme const&, double, void*) in weakperiodicbc.C.o
      oofem::PrescribedMean::giveInternalForcesVector(oofem::FloatArray&, oofem::TimeStep*, oofem::CharType, oofem::ValueModeType, oofem::UnknownNumberingScheme const&, oofem::FloatArray*, void*) in prescribedmean.C.o
      oofem::OctreeSpatialLocalizer::initElementIPDataStructure() in octreelocalizer.C.o
      _.omp_outlined..34 in engngm.C.o
      _.omp_outlined..64 in engngm.C.o
      ...
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
make[3]: *** [liboofem.dylib] Error 1
make[2]: *** [CMakeFiles/liboofem.dir/all] Error 2
make[1]: *** [CMakeFiles/oofem.dir/rule] Error 2
make: *** [oofem] Error 2

I think I should provide the location of the OpenMP library so the linking process goes successfully but I don't know where in the CMake Lists I should add it.

4 (edited by SaeidM 03-11-2021 10:02:30)

Re: OOFEM only using one single thread

Hi, the problem only occurs with the Apples's Clang compiler. Apple's Clang is more prone to errors compared to other compilers when it comes to OpenMP.
Switching to mainstream Clang compiler solves the problem.

Now I can run with multiple threads perfectly. Thanks for the guidance.

5 (edited by SaeidM 03-11-2021 11:25:50)

Re: OOFEM only using one single thread

In fact I think you might have forgotten to link the OpenMP library with oofem.
When I modify the CMakeLists.txt like this Apple's Clang can also compile the OOFEM with OpenMP.

if (USE_OPENMP)
    include (FindOpenMP)
    if (OPENMP_FOUND)
        set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
        set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
        set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
        list (APPEND MODULE_LIST "openmp")
        #
        ##this line is needed for Apple's Clang
        list (APPEND EXT_LIBS OpenMP::OpenMP_CXX)
        ##what I want to do is "target_link_libraries(oofem PRIVATE OpenMP::OpenMP_CXX)"
        #
    else()
       ##make sure the code will not compile despite the cmake options:
        message (FATAL_ERROR "OpenML library not found (required for multithreading)")
    endif ()
endif ()

This is also related.