Feature/gpu sharing factor

.github/workflows/basic.yml

@@ -80,7 +80,7 @@ jobs:
           -A "-W error" --no-coverage --fail-fast --durations 25
           --timings-json timings-${{ matrix.comms }}.json
       - name: Upload timings
-        if: always()
+        if: always() && env.ACT != 'true'
         uses: actions/upload-artifact@v7
         with:
           name: core-${{ matrix.comms }}-timings

.github/workflows/extra.yml

@@ -44,7 +44,7 @@ jobs:
           --tier core --no-coverage --fail-fast --durations 25
           --timings-json timings-core-${{ matrix.comms }}.json
       - name: Upload timings
-        if: always()
+        if: always() && env.ACT != 'true'
         uses: actions/upload-artifact@v7
         with:
           name: complete-core-${{ matrix.comms }}-timings
@@ -113,7 +113,7 @@ jobs:
           include-hidden-files: true
           if-no-files-found: warn
       - name: Upload timings
-        if: always()
+        if: always() && env.ACT != 'true'
         uses: actions/upload-artifact@v7
         with:
           name: coverage-timings-${{ matrix.shard }}
@@ -200,7 +200,7 @@ jobs:
           --shard ${{ matrix.shard }}/2
           --timings-json timings-external-${{ matrix.shard }}.json
       - name: Upload timings
-        if: always()
+        if: always() && env.ACT != 'true'
         uses: actions/upload-artifact@v7
         with:
           name: external-${{ matrix.shard }}-timings
@@ -227,7 +227,7 @@ jobs:
           --tier slow --no-coverage --fail-fast --durations 25
           --timings-json timings-slow.json
       - name: Upload timings
-        if: always()
+        if: always() && env.ACT != 'true'
         uses: actions/upload-artifact@v7
         with:
           name: slow-test-timings

AGENTS.md

@@ -86,6 +86,7 @@ for local development running individual tests is sufficient.
 - Individual unit tests can be run with ``pixi run -e dev pytest path/to/test_file``.
 - A libEnsemble run typically outputs an ``ensemble.log`` and ``libE_stats.txt`` file in the working directory. Check these files for tracebacks or run statistics.
 - An "ensemble" or "workflow" directory may also be created, often containing per-simulation output directories
+- Do not add tests solely to cover code that proves unreachable after analysis. If a branch/guard/except cannot be hit in production (e.g., defensive `None` checks before `set_rset_team()` or `try/except` around inputs already validated to `int|float|None`), prefer removing the dead code over manufacturing synthetic coverage (e.g., `__index__`/`__float__` tricks or `math.ceil` monkeypatches). Justify removal in the PR/commit.
 
 Modernizing Scripts for libEnsemble 2.0
 ---------------------------------------

docs/conf.py

@@ -128,6 +128,7 @@ class AxParameterWarning(Warning):  # Ensure it's a real warning subclass
     r"https://www\.sfu\.ca/~ssurjano/camel6\.html",
     r"https://github\.com/Libensemble/libensemble/blob/.*",
     r"https://img\.shields\.io/conda/v/conda-forge/libensemble",
+    r"https://github\.com/xopt-org/Xopt",
 ]
 
 autodoc_pydantic_model_show_json = False

docs/data_structures/libE_specs/libE_specs_resources.rst

@@ -18,17 +18,23 @@ Resources
 **num_resource_sets** [int]:
     The total number of resource sets into which resources will be divided.
     By default resources will be divided by workers (excluding
-    ``zero_resource_workers``).
+    ``zero_resource_workers``), or, if ``gpu_sharing_factor > 1``, into one
+    resource set per GPU share.
+
+**gpu_sharing_factor** [int] = ``1``:
+    Number of tasks that may share each GPU. For example, ``2`` lets two tasks
+    requesting ``num_gpus=0.5`` run on the same GPU. Tasks sharing a GPU share
+    its memory. See :ref:`GPU sharing<gpu-sharing>`.
 
 **gen_num_procs** [int] = ``0``:
     The default number of processors (MPI ranks) required by generators. Unless
     overridden by equivalent ``persis_info`` settings, generators will be allocated
     this many processors for applications launched via the MPIExecutor.
 
-**gen_num_gpus** [int] = ``0``:
+**gen_num_gpus** [int|float] = ``0``:
     The default number of GPUs required by generators. Unless overridden by
     the equivalent ``persis_info`` settings, generators will be allocated this
-    many GPUs.
+    many GPUs. May be fractional when ``gpu_sharing_factor > 1``.
 
 **gpus_per_group** [int]:
     Number of GPUs for each group in the scheduler. This can be used when

docs/resource_manager/overview.rst

@@ -186,6 +186,13 @@ will not be scheduled unless ``match_slots`` is set to *False*:
 
 This is only recommended if not enumerating resources to slot IDs (e.g., via ``CUDA_VISIBLE_DEVICES``).
 
+ **gpu_packing** [str]:
+    Where to place tasks that use part of a GPU when
+    :ref:`GPU sharing<gpu-sharing>` is enabled. ``"pack"`` fills partly used GPUs
+    and nodes first, keeping whole GPUs free for larger tasks. ``"spread"``
+    prefers the least used GPUs and nodes, giving tasks more room.
+    Default: ``"pack"``
+
 Note that if six resource sets were requested, then they would be split three per node, even
 if ``split2fit`` is *False*, as this could otherwise never be scheduled.
 
@@ -207,6 +214,138 @@ a number of resource sets.
 Note that persistent workers maintain their resources until they come out of a
 persistent state.
 
+.. _gpu-sharing:
+
+Sharing GPUs between tasks
+--------------------------
+
+Some simulations need only part of a GPU. ``gpu_sharing_factor`` lets several such
+tasks run on the same GPU at once, so a node's GPUs can serve more concurrent
+simulations.
+
+Set the number of tasks that may share each GPU, and have the generator request
+fractional GPUs through the ``num_gpus`` field:
+
+.. code-block:: python
+    :emphasize-lines: 2, 5
+
+    ensemble.libE_specs = LibeSpecs(
+        gpu_sharing_factor=2,  # Up to two tasks per GPU
+    )
+
+    ensemble.gen_specs = GenSpecs(outputs=[("x", float, 2), ("num_gpus", float)], ...)
+
+with the generator setting, for example:
+
+.. code-block:: python
+
+    H_o["num_gpus"] = [0.5, 1.0, 1.5, 2.0]
+
+GPUs are assigned and ``CUDA_VISIBLE_DEVICES`` (or equivalent) is set automatically,
+as with whole-GPU requests. There is no need to set ``num_resource_sets``.
+
+What tasks can request
+^^^^^^^^^^^^^^^^^^^^^^
+
+The sharing factor sets the step size of GPU requests. Requests are rounded up to
+the nearest step.
+
+.. list-table::
+    :header-rows: 1
+
+    * - ``gpu_sharing_factor``
+      - Smallest request
+      - Example requests
+    * - 1 (default)
+      - 1 GPU
+      - 1, 2, 4
+    * - 2
+      - 0.5 GPU
+      - 0.5, 1, 1.5, 2
+    * - 4
+      - 0.25 GPU
+      - 0.25, 0.5, 0.75, 1.25
+
+Requests can mix whole and partial GPUs. Each task gets its whole GPUs to itself.
+The fractional part of a request always comes from one GPU, never split across
+several. For example, with ``gpu_sharing_factor=4``, a task requesting ``0.75`` GPUs
+gets three quarters of one GPU, not shares of two or three different GPUs.
+
+With ``gpu_sharing_factor=2``, a task requesting ``1.5`` GPUs gets one GPU to itself
+and half of a second GPU, which it may share with one other task. It sees two GPUs.
+
+Generators can also use a fraction of a GPU, via ``gen_num_gpus``:
+
+.. code-block:: python
+
+    ensemble.libE_specs = LibeSpecs(gpu_sharing_factor=4, gen_num_gpus=0.25)
+
+Example: a node with four GPUs
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+With ``gpu_sharing_factor=2`` on a node with four GPUs, any of these sets of tasks
+can run at once (using the default ``"pack"`` placement, below):
+
+* Eight tasks requesting ``0.5`` GPUs (two per GPU).
+* Four tasks requesting ``0.5`` GPUs and two requesting ``1`` GPU.
+* Two tasks requesting ``1.5`` GPUs and two requesting ``0.5`` GPUs.
+
+Run with enough workers to cover the concurrency you want. For eight concurrent
+half-GPU tasks per node, use eight workers per node.
+
+Choosing a sharing factor
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Tasks sharing a GPU are not isolated: they share its memory and compute. Choose a
+factor so that this many tasks fit in one GPU's memory at once. For example, with
+40 GB GPUs and tasks needing about 8 GB, a factor of up to ``4`` is safe.
+
+If your platform provides tools such as NVIDIA MPS for better GPU sharing, they can
+be enabled as usual alongside ``gpu_sharing_factor``.
+
+Each task's share of CPU cores also gets smaller: libEnsemble divides each node's
+cores between all GPU shares. For example, on a node with 32 cores and four GPUs,
+a factor of ``2`` gives four cores per share.
+
+Placing partial-GPU tasks
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+By default, tasks using part of a GPU are packed onto GPUs that are already partly
+used, keeping whole GPUs free for larger tasks. To instead spread such tasks over
+the least used GPUs and nodes:
+
+.. code-block:: python
+
+    ensemble.libE_specs = LibeSpecs(
+        gpu_sharing_factor=2,
+        scheduler_opts={"gpu_packing": "spread"},
+    )
+
+Use ``"pack"`` (the default) when tasks of different sizes are mixed. With
+``"spread"``, four half-GPU tasks on a four-GPU node take one share of every GPU,
+so a following task requesting a whole GPU must wait. Use ``"spread"`` when tasks
+are all the same size, or when they compete for memory or bandwidth and should be
+kept apart.
+
+Resource sets and errors
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+When ``gpu_sharing_factor`` is greater than 1 and ``num_resource_sets`` is not set,
+libEnsemble uses one resource set per GPU share and logs a message such as::
+
+    gpu_sharing_factor=2: num_resource_sets defaulting to 16 (2 node(s) x 4 GPUs x 2 shares)
+
+If you do set ``num_resource_sets``, it must provide at least one resource set per
+GPU share on each node (for example, at least ``16`` in the case above). Fewer
+raises an error. More is allowed, with the extra resource sets having no GPUs.
+
+An error is also raised if the factor would leave a resource set with no CPU cores.
+
+libEnsemble warns at startup if ``gpu_sharing_factor`` is set but tasks do not
+request GPUs, or if no GPUs are detected.
+
+For an example, see `test_GPU_sharing_variable_resources.py`_.
+
 Example scenarios
 -----------------
 
@@ -270,6 +409,7 @@ Also, this can be set on the command line as a convenience.
     python run_ensemble.py --nworkers 5 --nresource_sets 8
 
 .. _persistent_sampling_var_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/gen_funcs/persistent_sampling_var_resources.py
+.. _test_GPU_sharing_variable_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_sharing_variable_resources.py
 .. _test_GPU_variable_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/regression_tests/test_GPU_variable_resources.py
 .. _test_persistent_sampling_CUDA_variable_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/tests/functionality_tests/test_persistent_sampling_CUDA_variable_resources.py
 .. _var_resources.py: https://github.com/Libensemble/libensemble/blob/develop/libensemble/sim_funcs/var_resources.py

libensemble/executors/executor.py

@@ -79,18 +79,18 @@ class Application:
 
     def __init__(
         self,
-        full_path: str,
+        full_path: str | os.PathLike[str],
         name: str | None = None,
         calc_type: str | None = "sim",
         desc: str | None = None,
         pyobj: Any | None = None,
-        precedent: str = "",
+        precedent: str | os.PathLike[str] = "",
     ) -> None:
         """Instantiates a new Application instance."""
-        self.full_path = full_path
+        self.full_path = os.fspath(full_path)
         self.calc_type = calc_type
-        self.calc_dir, self.exe = os.path.split(full_path)
-        self.precedent = precedent
+        self.calc_dir, self.exe = os.path.split(self.full_path)
+        self.precedent = os.fspath(precedent) if precedent else ""
 
         if self.exe.endswith(".py"):
             if not precedent:

libensemble/executors/mpi_runner.py

@@ -1,5 +1,6 @@
 import argparse
 import logging
+import math
 
 from libensemble.executors.executor import jassert
 from libensemble.resources import mpi_resources
@@ -124,8 +125,8 @@ def _set_gpu_cli_option(self, wresources, extra_args, gpu_setting_name, gpu_valu
     def _set_gpu_env_var(self, wresources, task, gpus_per_node, gpus_env):
         """Add GPU environment variable setting to the tasks environment"""
         jassert(wresources.matching_slots, f"Cannot assign CPUs/GPUs to non-matching slots per node {wresources.slots}")
-        slot_list = wresources.get_slots_as_string(multiplier=wresources.gpus_per_rset_per_node, limit=gpus_per_node)
-        task._add_to_env(gpus_env, slot_list)
+        gpu_list = wresources.get_gpu_ids_as_string(limit=gpus_per_node)
+        task._add_to_env(gpus_env, gpu_list)
 
     def _local_runner_set_gpus(self, task, wresources, extra_args, gpus_per_node, ppn):
         """Set default GPU setting for MPI runner"""
@@ -174,7 +175,7 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args,
 
         # gpus per node for this worker.
         if wresources.doihave_gpus():
-            gpus_avail_per_node = wresources.slot_count * wresources.gpus_per_rset_per_node
+            gpus_avail_per_node = len(set(wresources.gpu_ids_on_node)) * wresources.gpus_per_rset_per_node
         else:
             gpus_avail_per_node = 0
 
@@ -188,7 +189,7 @@ def _assign_gpus(self, task, resources, nprocs, nnodes, ppn, ngpus, extra_args,
                 nnodes = wresources.local_node_count
 
         if ngpus is not None:
-            gpus_req_per_node = ngpus // nnodes
+            gpus_req_per_node = ngpus // nnodes if isinstance(ngpus, int) else math.ceil(ngpus / nnodes)
             if gpus_req_per_node > gpus_avail_per_node:
                 logger.info(f"Asked for more GPUs per node than available - max is {gpus_avail_per_node}")
             gpus_per_node = min(gpus_req_per_node, gpus_avail_per_node)
@@ -246,8 +247,8 @@ def _get_min_nodes(self, nprocs, ppn, nnodes, ngpus, resources):
             if nprocs:
                 proc_min_nodes = (nprocs + procs_on_node - 1) // procs_on_node
             if ngpus:
-                gpus_on_node = wresources.slot_count * wresources.gpus_per_rset_per_node
-                gpu_min_nodes = (ngpus + gpus_on_node - 1) // gpus_on_node
+                gpus_on_node = len(set(wresources.gpu_ids_on_node)) * wresources.gpus_per_rset_per_node
+                gpu_min_nodes = math.ceil(ngpus / gpus_on_node) if gpus_on_node else 1
 
             min_nodes = max(proc_min_nodes, gpu_min_nodes)
             nnodes = min(min_nodes, total_nodes)
@@ -261,6 +262,9 @@ def _adjust_procs(self, nprocs, ppn, nnodes, ngpus, resources):
 
         def adjust_resource(n_units, units_attr, units_name):
             if n_units is not None and nnodes:
+                # Fractional GPU requests are rounded up per node in _assign_gpus
+                if isinstance(n_units, float) and not n_units.is_integer():
+                    return n_units
                 mod_n_units = n_units % nnodes
                 if mod_n_units != 0:
                     try_n_units = n_units + (nnodes - mod_n_units)

libensemble/manager.py

@@ -279,6 +279,9 @@ def __init__(
         if self.resources is not None:
             gresource = self.resources.glob_resources
             self.scheduler_opts = gresource.update_scheduler_opts(self.scheduler_opts)
+            if self.resources.resource_manager is not None and self.resources.resource_manager.rsets_sizing_msg:
+                logger.info(self.resources.resource_manager.rsets_sizing_msg)
+            self._check_gpu_sharing(gresource)
 
         for wrk in self.W:
             if wrk["worker_id"] in self.libE_specs.get("gen_workers", []):
@@ -296,6 +299,20 @@ def __init__(
                 e,
             )
 
+    def _check_gpu_sharing(self, gresource) -> None:
+        """Warn about gpu_sharing_factor configurations that will not share GPUs as intended"""
+        sf = gresource.gpu_sharing_factor
+        if sf <= 1:
+            return
+        if not gresource.gpus_avail_per_node:
+            logger.manager_warning(f"gpu_sharing_factor={sf} is set, but no GPUs were detected. It will be ignored.")
+        elif "num_gpus" not in self.hist.H.dtype.names and not self.gen_num_gpus:
+            logger.manager_warning(
+                f"gpu_sharing_factor={sf} is set, but tasks do not request GPUs. For best GPU utilization, "
+                "tasks should request GPUs via a 'num_gpus' field in gen_specs['out'] (or 'gen_num_gpus' "
+                "for generators). Without GPU requests, tasks are not placed by GPU share and GPUs may be underused."
+            )
+
     # --- Termination logic routines
 
     def term_test_wallclock(self, max_elapsed: int) -> bool:
@@ -1116,15 +1133,11 @@ def run(self, persis_info: dict) -> tuple[dict, int, int]:
                             if self._service_idle_start is None:
                                 self._service_idle_start = time.time()
                             elif time.time() - self._service_idle_start > idle_timeout:
-                                logger.info(
-                                    f"service_mode idle for {idle_timeout}s with no work, exiting"
-                                )
+                                logger.info(f"service_mode idle for {idle_timeout}s with no work, exiting")
                                 break
                         time.sleep(0.1)
                         continue
-                    raise AssertionError(
-                        "alloc_f did not return any work, although all workers are idle."
-                    )
+                    raise AssertionError("alloc_f did not return any work, although all workers are idle.")
                 # work was dispatched OR workers active OR terminating — reset idle clock
                 self._service_idle_start = None
         except WorkerException as e:

libensemble/resources/resources.py

@@ -164,6 +164,7 @@ def __init__(self, libE_specs: dict, platform_info: dict = {}, top_level_dir: st
         self.num_resource_sets = libE_specs.get("num_resource_sets", None)
         self.enforce_worker_core_bounds = libE_specs.get("enforce_worker_core_bounds", False)
         self.gpus_per_group = libE_specs.get("gpus_per_group")
+        self.gpu_sharing_factor = libE_specs.get("gpu_sharing_factor", 1)
         resource_info = libE_specs.get("resource_info", {})
 
         # resource_info overrides platform