Example Site Configurations

[sites.piz_daint]
    site_type = "slurm"
    default_ranks = 12
    max_ranks = 10000
    salvus_binary = "/users/{USER}/Salvus/bin/salvus"
    run_directory = "/scratch/snx3000/{USER}/salvus_flow/run"
    tmp_directory = "/scratch/snx3000/{USER}/salvus_flow/tmp"
    # The following line enables GPU support on Piz daint
    # Note that, CUDA kernels for adjoint simulations are not included in
    # the current release, and you need to set the value to `false` if you
    # want to compute gradients.
    use_cuda_capable_gpus = true
    [[sites.piz_daint.environment_variable]]
        name = "CRAY_CUDA_MPS"
        value = "1"
    # The cray-mpich-abi module does not set the LD_LIBRARY_PATH
    # so it has to be done manually. In case this no longer works, you can find
    # the correct path by looking for the CRAY_LD_LIBRARY_PATH setting in
    # `module show cray-mpich-abi`.
    [[sites.piz_daint.environment_variable]]
        name = "LD_LIBRARY_PATH"
        value = "/opt/cray/pe/mpt/7.7.18/gni/mpich-gnu-abi/8.2/lib"
    [sites.piz_daint.ssh_settings]
        hostname = "daint.cscs.ch"
        username = "{USER}"
    [sites.piz_daint.site_specific]
        tasks_per_node = 12
        partition = "normal"
        debug_partition = "debug"
        path_to_slurm_binaries = "/usr/bin"
        # Nov 2019: This is a workaround for a bug in the slurm version the
        # CSCS deployed. They are aware of it and are working on a fix.
        omit_default_srun_arguments = true
        # These are account/project dependent!
        [[sites.piz_daint.site_specific.additional_sbatch_arguments]]
            name = "constraint"
            value = "gpu"
        [[sites.piz_daint.site_specific.additional_sbatch_arguments]]
            name = "account"
            value = "{ACCOUNT}"
        # You have to switch two modules. The first switch is just because the
        # default module is incompatible with the second module we want to
        # load.
        [[sites.piz_daint.site_specific.modules_to_switch]]
            old = "PrgEnv-cray"
            new = "PrgEnv-gnu"
        # Load an ABI compatible MPI module for Salvus to use.
        [[sites.piz_daint.site_specific.modules_to_switch]]
            old = "cray-mpich"
            new = "cray-mpich-abi"

[sites.piz_daint_mc]
    site_type = "slurm"
    default_ranks = 12
    max_ranks = 10000
    salvus_binary = "/users/{USER}/Salvus/bin/salvus"
    run_directory = "/scratch/snx3000/{USER}/salvus_flow/run"
    tmp_directory = "/scratch/snx3000/{USER}/salvus_flow/tmp"
    # The cray-mpich-abi module does not set the LD_LIBRARY_PATH
    # so it has to be done manually. In case this no longer works, you can find
    # the correct path by looking for the CRAY_LD_LIBRARY_PATH setting in
    # `module show cray-mpich-abi`.
    [[sites.piz_daint_mc.environment_variable]]
        name = "LD_LIBRARY_PATH"
        value = "/opt/cray/pe/mpt/7.7.18/gni/mpich-gnu-abi/8.2/lib"
    [sites.piz_daint_mc.ssh_settings]
        hostname = "daint.cscs.ch"
        username = "{USER}"
    [sites.piz_daint_mc.site_specific]
        tasks_per_node = 12
        partition = "normal"
        debug_partition = "debug"
        path_to_slurm_binaries = "/usr/bin"
        # Nov 2019: This is a workaround for a bug in the slurm version the
        # CSCS deployed. They are aware of it and are working on a fix.
        omit_default_srun_arguments = true
        # These are account/project dependent!
        [[sites.piz_daint_mc.site_specific.additional_sbatch_arguments]]
            name = "constraint"
            value = "mc"
        [[sites.piz_daint_mc.site_specific.additional_sbatch_arguments]]
            name = "account"
            value = "{ACCOUNT}"
        # You have to switch two modules. The first switch is just because the
        # default module is incompatible with the second module we want to
        # load.
        [[sites.piz_daint_mc.site_specific.modules_to_switch]]
            old = "PrgEnv-cray"
            new = "PrgEnv-gnu"
        # Load an ABI compatible MPI module for Salvus to use.
        [[sites.piz_daint_mc.site_specific.modules_to_switch]]
            old = "cray-mpich"
            new = "cray-mpich-abi"

[sites.eejit]
    site_type = "slurm"
    default_ranks = 48
    max_ranks = 960
    salvus_binary = "/quanta1/home/{USER}/Salvus/bin/salvus"
    run_directory = "/scratch/{USER}/SalvusFlow/run"
    tmp_directory = "/scratch/{USER}/SalvusFlow/tmp"
    [sites.eejit.ssh_settings]
        hostname = "eejit.geo.uu.nl"
        username = "{USER}"
    [sites.eejit.site_specific]
        tasks_per_node = 48
        partition = "gpu"
        path_to_slurm_binaries = "/usr/bin"
        omit_default_srun_arguments = true
        replace_srun_with = "mpirun"
        modules_to_load = ["intel-mpi/64/2018.0.128"]
        [[sites.eejit.site_specific.additional_srun_arguments]]
            name = "n"
            value = "$SLURM_NTASKS"

[sites.supermuc-ng]
    site_type = "slurm"
    default_ranks = 48
    # 16 nodes - the maximum for the micro partition.
    max_ranks = 768
    # Depends on where you put the Salvus binary.
    salvus_binary = "/hppfs/work/{ACCOUNT}/{USER}/Salvus/bin/salvus"
    # Small to medium files are stored in the work are of the file system.
    run_directory = "/hppfs/work/{ACCOUNT}/{USER}/salvus_data/run"
    # Larger temporary files (for example checkpoints for adjoint runs) might
    # be better stored in the scratch space. Per default 1 PB/user which should
    # be plenty. Remember that the files are automatically deleted after a few
    # weeks.
    tmp_directory = "/hppfs/scratch/{HASH}/{USER}/salvus_data/tmp"
    # No internet access on SuperMUC-NG.
    use_license_tokens = true
    [sites.supermuc-ng.ssh_settings]
        hostname = "skx.supermuc.lrz.de"
        username = "{USER}"
    [sites.supermuc-ng.site_specific]
        # SuperMUC-NG has 48 cores per node. This settings will be used to
        # compute the number of nodes the final job will run on.
        tasks_per_node = 48
        # The 'micro' partition is used for 1-16 nodes, use 'general' for jobs
        # using 17-768 nodes.
        partition = "micro"
        debug_partition = "test"
        path_to_slurm_binaries = "/usr/bin"
        # The account must be set.
        [[sites.supermuc-ng.site_specific.additional_sbatch_arguments]]
            name = "account"
            value = "{ACCOUNT}"

[sites.euler_slurm]
        site_type = "slurm"
        default_ranks = 12
        # Specify this depending on your allowance on Euler.
        max_ranks = 640
        # This is just the default place at which the downloader will put Salvus.
        salvus_binary = "/cluster/home/{USER}/Salvus/bin/salvus"
        # As always - make sure these directories are fine for you. You might have
        # access to more suitable folders on Euler, depending on your project but
        # every Euler user has access to these.
        run_directory = "/cluster/scratch/{USER}/salvus_data/run"
        tmp_directory = "/cluster/scratch/{USER}/salvus_data/tmp"
        # Compute nodes don't have internet access, so use license tokens.
        use_license_tokens = true
        [sites.euler_slurm.ssh_settings]
            hostname = "euler.ethz.ch"
            username = "{USER}"
        [sites.euler_slurm.site_specific]
            # Euler maintains two different module systems
            # https://scicomp.ethz.ch/wiki/Modules_and_applications
            # Make sure to use the more modern "LMOD Modules", which comes with a
            # newer version of Intel MPI.
            modules_to_load = ['intel']
            # Euler has different partitions depending on the maximum walltime of
            # a job. For longer jobs, consider switching to 'normal.24h' or
            # 'normal.72h'.
            partition = "normal.4h"
            # Euler_VIII is equipped with 64-core AMD EPYC 7742 processors.
            # https://scicomp.ethz.ch/wiki/Euler#Euler_VIII
            tasks_per_node = 64
            path_to_slurm_binaries = "/cluster/apps/slurm/bin/"

[sites.gadi]
        site_type = "pbs"
        # This depends on the chosen Gadi queue.
        default_ranks = 48
        # Just a safety measure.
        max_ranks = 800
        # Adapt these to your folders!
        salvus_binary = "/home/_XXX_/_USER_/Salvus/bin/salvus"
        run_directory = "/scratch/_PROJECT_/_USER_/salvus_flow/run"
        tmp_directory = "/scratch/_PROJECT_/_USER_/salvus_flow/tmp"
        # The compute nodes on gadi cannot access the internet, so license
        # tokens have to be used. The jobs will hang if you don't set this to
        # true.
        use_license_tokens = true
        # Make sure the set up key based SSH authentication beforehand as
        # described in the Salvus documentation.
        [sites.gadi.ssh_settings]
            hostname = "gadi.nci.org.au"
            username = "_USER_"
        # Point to the releases folder in the chosen intel MPI module as that
        # is where the required ABI compatible MPI libraries are located.
        [[sites.gadi.environment_variable]]
            name = "LD_LIBRARY_PATH"
            value = "/apps/intel-mpi/2021.4.0/lib/release"
        [sites.gadi.site_specific]
            tasks_per_node = 48
            # Gadi has 192GB memory per node so it cannot hurt to have a bunch
            # here. The NCI starts charging more when the memory per rank is
            # over 4 GB as it then considers it a memory dominated request.
            memory_per_rank_in_mb = 2048
            # Normal queue for most things, express for init-site.
            queue = "normal"
            debug_queue = "express"
            compute_resources_template = "ncpus={RANKS}"
            path_to_pbs_binaries = "/opt/pbs/default/bin"
            # Use the mpirun from the intel-mpi module.
            replace_pbsrun_with = "mpirun"
            # Salvus requires an ABI compatible MPI.
            modules_to_load = ["intel-mpi/2021.4.0"]
            # Storage and project have to explicitly passed. If you run salvus
            # elsewhere you might to add some other folder here.
            [[sites.gadi.site_specific.additional_qsub_arguments]]
                name = "l"
                value = "storage=scratch/_PROJECT_"
            [[sites.gadi.site_specific.additional_qsub_arguments]]
                name = "P"
                value = "_PROJECT_"

[sites.euler]
    # Euler uses IBM's LSF system.
    site_type = "lsf"
    # Node size.
    default_ranks = 24
    # Specify this depending on your allowance on euler.
    max_ranks = 240
    # This is just the default place at which the downloader will put Salvus.
    salvus_binary = "/cluster/home/{USER}/Salvus/bin/salvus"
    # As always - make sure these directories are fine for you. You might have
    # access to more suitable folders on Euler, depending on your project but
    # every Euler user has access to these.
    run_directory = "/cluster/home/{USER}/salvus_flow_run"
    tmp_directory = "/cluster/scratch/{USER}/salvus_flow_temp"
    # Compute nodes don't have internet access, so use license tokens.
    use_license_tokens = true
    [sites.euler.ssh_settings]
        hostname = "euler.ethz.ch"
        username = "{USER}"
    [sites.euler.site_specific]
        # Euler maintains two different module systems
        # https://scicomp.ethz.ch/wiki/Modules_and_applications
        # The following setting is different for each configuration.
        #
        # When using the "Environment Modules", first load the `new` module which
        # gives access to a lot more modules. Then load Intel's MPI which is an ABI
        # compatible MPI that works with Salvus.
        modules_to_load = ['new', 'impi']
        # Comment out the line above and use the following line instead when working
        # with the "LMOD Modules".
        # modules_to_load = ['intel']
        # `fullnode` is a special resource specifier on Euler which grants
        # exclusive access to a single node. If not given jobs from other
        # users might run on the same site.
        #
        # This configuration sets it to only be set for jobs with 18 or more
        # ranks.
        [[sites.euler.site_specific.additional_bsub_argument]]
            name = "R"
            value = "fullnode"
            condition = "ranks >= 18"
        # Euler by defaults only grants 1024 MB of RAM per rank. This setting
        # doubles that.
        [[sites.euler.site_specific.additional_bsub_argument]]
            name = "R"
            value = "rusage[mem=2048]"

[sites.glic]
    site_type = "lsf"
    default_ranks = 12
    max_ranks = 120
    # Path to the downloaded binary.
    salvus_binary = "/home/{USER}/Salvus/bin/salvus"
    run_directory = "/home/{USER}/salvus_data/run"
    tmp_directory = "/home/{USER}/salvus_data/tmp"
    # The LSF binaries are not on its default path. Thus, set the path
    # here. The actual location is likely to change in the future.
    [[sites.glic.environment_variable]]
        name = "PATH"
        value = "/lsf/10.1/linux3.10-glibc2.17-x86_64/bin:$PATH"
    # LSF requires a few environment variables which are not set when not
    # logging in in a shell. Thus they are set here. Please note that the
    # actual location for these might also change with time. To figure the
    # current values out just log-in to GLIC and query them.
    [[sites.glic.environment_variable]]
        name = "LSF_ENVDIR"
        value = "/lsf/conf"
    [[sites.glic.environment_variable]]
        name = "LSF_SERVERDIR"
        value = "/lsf/10.1/linux3.10-glibc2.17-x86_64/etc"
    # This example use the Mondaic provided MPI. Thus point to its libraries.
    [[sites.glic.environment_variable]]
        name = "LD_LIBRARY_PATH"
        value = "/home/{USER}/Salvus/lib"
    # SSH settings.
    [sites.glic.ssh_settings]
        hostname = "{HOSTNAME}"
        username = "{USER}"
    # LSF specific settings.
    [sites.glic.site_specific]
        # Use Mondaic's provided MPI as it seem to work fine here.
        replace_mpirun_with = "/home/{USER}/Salvus/bin/mpirun"
        # Use your group's queue.
        queue = "{QUEUE}"
        # Load the latest intel module and tell Salvus to load the module in
        # the job submission file and not before.
        modules_to_load = ["intel/intel-2019u3"]
        change_modules_in_batch_file = true

[sites.arc4]
    site_type = "grid_engine"
    # Potentially a bit much but this is the size of one node.
    default_ranks = 40
    # Safety measure - feel free to set this higher.
    max_ranks = 400
    # Path to the Salvus binary. Feel free to move this elsewhere.
    salvus_binary = "/home/home02/{USERNAME}/Salvus/bin/salvus"
    # Use the /nobackup mounted parallel file system.
    run_directory = "/nobackup/{USERNAME}/salvus_data/run"
    tmp_directory = "/nobackup/{USERNAME}/salvus_data/tmp"

    [sites.arc4.ssh_settings]
        # Could be the full hostname here or a reference to the SSH config
        # which is necessary to setup up proxy jump hosts.
        hostname = "arc4"
        username = "{USER}"

    [sites.arc4.site_specific]
        tasks_per_node = 40
        mpirun_command = "mpirun"
        # Salvus by default executes in bash - so the openmpi module has to be
        # unloaded.
        modules_to_unload = ["openmpi"]
        # Load a compatible PI.
        modules_to_load = ["intel", "mvapich2"]

        # Request 4GB of memory per rank. Can be adjusted.
        [[sites.arc4.site_specific.additional_qsub_arguments]]
        name = "l"
        value = "h_vmem=4G"

        # This optional setting causes it to request only whole nodes for jobs
        # with 20 or more ranks.
        [[sites.arc4.site_specific.additional_qsub_arguments]]
        name = "l"
        value = "np={RANKS}"
        condition = "ranks >= 20"

        # This optional setting results in at max 20 parallel jobs for job array
        # runs.
        [[sites.arc4.site_specific.additional_qsub_arguments]]
        name = "tc"
        value = "20"
        condition = "is_job_array"

[sites.arc4_gpu]
    site_type = "grid_engine"
    # Single GPU - can of course use more if they are available.
    default_ranks = 1
    max_ranks = 10
    # Path to the Salvus binary. Feel free to move this elsewhere.
    salvus_binary = "/home/home02/{USERNAME}/Salvus/bin/salvus"
    # Use the /nobackup mounted parallel file system.
    run_directory = "/nobackup/{USERNAME}/salvus_data/run"
    tmp_directory = "/nobackup/{USERNAME}/salvus_data/tmp"

    # Use GPUs of course!
    use_cuda_capable_gpus = true

    [sites.arc4_gpu.ssh_settings]
        # Could be the full hostname here or a reference to the SSH config
        # which is necessary to setup up proxy jump hosts.
        hostname = "arc4"
        username = "{USER}"

    [sites.arc4_gpu.site_specific]
        tasks_per_node = 10
        mpirun_command = "mpirun"
        modules_to_unload = ["openmpi"]
        modules_to_load = ["intel", "mvapich2"]

        # Request a single GPU and 10 cores.
        [[sites.arc4_gpu.site_specific.additional_qsub_arguments]]
        name = "l"
        value = "coproc_v100=1"