Enable Metal if available

GPU/GPUTracking/CMakeLists.txt

@@ -509,9 +509,7 @@ if(CPUDEVICE_ENABLED)
 endif()
 
 if(ALIGPU_BUILD_TYPE STREQUAL "O2" OR ALIGPU_BUILD_TYPE STREQUAL "Standalone")
-  if(NOT ${CMAKE_SYSTEM} MATCHES Darwin)
-    add_subdirectory(Standalone/Benchmark)
-  endif()
+  add_subdirectory(Standalone/Benchmark)
 endif()
 
 if(GPUCA_DETERMINISTIC_MODE GREATER_EQUAL ${GPUCA_DETERMINISTIC_MODE_MAP_GPU})

GPU/GPUTracking/Definitions/GPUSettingsList.h

@@ -355,8 +355,8 @@ AddOption(keepAllMemory, bool, false, "", 0, "Allocate all memory on both device
 AddOption(keepDisplayMemory, bool, false, "", 0, "Like keepAllMemory, but only for memory required for event display")
 AddOption(disableMemoryReuse, bool, false, "", 0, "Disable memory reusage (for debugging only)")
 AddOption(memoryAllocationStrategy, int8_t, 0, "", 0, "Memory Allocation Strategy (0 = auto, 1 = individual allocations, 2 = single global allocation)")
-AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory (0 = disable, 1 / 2 = auto, other values set the size of the memory pool)", min(0ul))
-AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(0ul))
+AddOption(forceMemoryPoolSize, uint64_t, 1, "memSize", 0, "Force size of allocated GPU / page locked host memory (0 = disable, 1 / 2 = auto, other values set the size of the memory pool)", min(uint64_t(0)))
+AddOption(forceHostMemoryPoolSize, uint64_t, 0, "hostMemSize", 0, "Force size of allocated host page locked host memory (overriding memSize)", min(uint64_t(0)))
 AddOption(memoryScalingFactor, float, 1.f, "", 0, "Factor to apply to all memory scalers")
 AddOption(memoryScalingFuzz, uint64_t, 0, "", 0, "Fuzz the memoryScalingFactor (0 disable, 1 enable, >1 set seed", def(1))
 AddOption(tpcInputWithClusterRejection, uint8_t, 0, "", 0, "Indicate whether the TPC input is CTF data with cluster rejection, to tune buffer estimations")
@@ -610,7 +610,7 @@ EndConfig()
 BeginConfig(GPUSettingsStandalone, configStandalone)
 AddOption(runGPU, uint8_t, 1, "", 'g', "Use GPU for processing", message("GPU processing enabled"), set(2))
 AddOptionSet(runGPU, uint8_t, 0, "", 'c', "Use CPU for processing", message("CPU enabled"))
-AddOption(gpuType, std::string, "AUTO", "", 0, "GPU type (CUDA / HIP / OCL / CPUDEVICE) or CPU or AUTO")
+AddOption(gpuType, std::string, "AUTO", "", 0, "GPU type (CUDA / HIP / OCL / METAL / CPUDEVICE) or CPU or AUTO")
 AddOption(runGPUforce, bool, true, "", 0, "Force usage of the specified GPU device type, no CPU fallback")
 AddOption(promptOnExit, bool, false, "", 0, "Prompt for keypress before exiting")
 AddOption(continueOnError, bool, false, "", 0, "Continue processing after an error")
@@ -627,8 +627,8 @@ AddOption(eventDisplay, int32_t, 0, "display", 'd', "Show standalone event displ
 AddOption(eventGenerator, bool, false, "", 0, "Run event generator")
 AddOption(cont, bool, false, "", 0, "Process continuous timeframe data, even if input is triggered")
 AddOption(setMaxTimeBin, int32_t, -2, "", 0, "maximum time bin of continuous data, 0 for triggered events, -1 for automatic continuous mode, -2 for automatic continuous / triggered")
-AddOption(outputcontrolmem, uint64_t, 0, "outputMemory", 0, "Use predefined output buffer of this size", min(0ul), message("Using %s bytes as output memory"))
-AddOption(inputcontrolmem, uint64_t, 0, "inputMemory", 0, "Use predefined input buffer of this size", min(0ul), message("Using %s bytes as input memory"))
+AddOption(outputcontrolmem, uint64_t, 0, "outputMemory", 0, "Use predefined output buffer of this size", min(uint64_t(0)), message("Using %s bytes as output memory"))
+AddOption(inputcontrolmem, uint64_t, 0, "inputMemory", 0, "Use predefined input buffer of this size", min(uint64_t(0)), message("Using %s bytes as input memory"))
 AddOption(cpuAffinity, int32_t, -1, "", 0, "Pin CPU affinity to this CPU core", min(-1))
 AddOption(fifoScheduler, bool, false, "", 0, "Use FIFO realtime scheduler", message("Setting FIFO scheduler: %s"))
 AddOption(fpe, int8_t, -1, "", 0, "Trap on floating point exceptions (-1 = if no ffast-math)")

GPU/GPUTracking/Standalone/Benchmark/standalone.cxx

@@ -116,6 +116,7 @@ int32_t ReadConfiguration(int argc, char** argv)
   setlocale(LC_ALL, "en_US.utf-8");
   setlocale(LC_NUMERIC, "en_US.utf-8");
   if (configStandalone.cpuAffinity != -1) {
+#ifdef __linux__
     cpu_set_t mask;
     CPU_ZERO(&mask);
     CPU_SET(configStandalone.cpuAffinity, &mask);
@@ -125,8 +126,13 @@ int32_t ReadConfiguration(int argc, char** argv)
       printf("Error setting CPU affinity\n");
       return 1;
     }
+#else
+    printf("CPU affinity not supported on this platform\n");
+    return 1;
+#endif
   }
   if (configStandalone.fifoScheduler) {
+#ifdef __linux__
     printf("Setting FIFO scheduler\n");
     sched_param param;
     sched_getparam(0, &param);
@@ -135,13 +141,21 @@ int32_t ReadConfiguration(int argc, char** argv)
       printf("Error setting scheduler\n");
       return 1;
     }
+#else
+    printf("FIFO scheduler not supported on this platform\n");
+    return 1;
+#endif
   }
 #ifdef __FAST_MATH__
   if (configStandalone.fpe == 1) {
 #else
   if (configStandalone.fpe) {
 #endif
+#ifdef __linux__
     feenableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW);
+#else
+    printf("Trapping floating point exceptions not supported on this platform\n");
+#endif
   }
   bool detMode = false, noFTZMode = false;
 #ifdef GPUCA_DETERMINISTIC_MODE
@@ -1001,7 +1015,9 @@ int32_t main(int argc, char** argv)
 
 #ifndef _WIN32
   if (configStandalone.proc.runQA && configStandalone.fpe) {
+#ifdef __linux__
     fedisableexcept(FE_INVALID | FE_DIVBYZERO | FE_OVERFLOW);
+#endif
   }
 #endif
 

GPU/GPUTracking/qa/genEvents.cxx

@@ -357,7 +357,7 @@ void genEvents::RunEventGenerator(GPUChainTracking* rec, const std::string& dir)
 {
   std::unique_ptr<genEvents> gen(new genEvents(rec));
   mkdir(dir.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
-  rec->DumpSettings(dir.c_str());
+  rec->rec()->DumpSettings(dir.c_str()); // the chain's own override does not write settings.dump
 
   gen->InitEventGenerator();
 

GPU/documentation/build-standalone.md

@@ -81,18 +81,42 @@ The standalone benchmark supports running on Run2 data exported from AliRoot, or
 In o2, `o2-tpc-reco-workflow` and the `o2-gpu-reco-workflow` can dump event data with the `configKeyValue` `GPU_global.dump=1;`.
 This will dump the event data to the local folder, all dumped files have a `.dump` file extension. If there are multiple TFs/events processed, there will be multiple `event.i.dump` files. In order to create a standalone dataset out of these, just copy all the `.dump` files to a subfolder in `~/standalone/events/[FOLDERNAME]`.
 
+## What a dataset consists of
+
+A dataset folder is read by `GPUReconstruction::ReadSettings` and `GPUChainTracking::ReadSettings`, and holds:
+
+- `settings.dump` - the GRP settings (solenoid field, continuous max time bin). Without it the benchmark aborts with `Error reading event config file`.
+- `event.<i>.dump` - one file per time frame / event.
+- the calibration objects, each written only if it was present when dumping: `tpctransform.dump`, `tpcpadgaincalib.dump`, `tpczslinkmapping.dump`, `dEdxCalibContainer.dump`, `matlut.dump`, `trdgeometry.dump`, `trdrecoparam.dump`.
+
+`tpctransform.dump` is needed for everything except already-transformed clusters; without it the run stops at `Need cluster native data for on-the-fly TPC transform`. `dEdxCalibContainer.dump` is needed unless dE/dx is switched off with `--rundEdx 0`.
+
+The folder is looked up under `./events/` relative to the working directory, so `-e mydata` reads `./events/mydata/`. Use `--absoluteEventsDir` to pass a path outside of `./events/`.
+
+## From simulated data
+
 Data can be dumped from raw data, or from MC data, e.g. generated by the Full System Test. In case of MC data, also MC labels are dumped, such that they are used in the `./ca --qa` mode.
 
-To get a dump from simulated data, please run e.g. the FST simulation as described in [full-system-test-setup.md](https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/full-system-test-setup.md).
-A simple run as
+First simulate, digitize and convert to raw, as described in [full-system-test-setup.md](https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/full-system-test-setup.md). Run it in a scratch folder, since it writes a lot of intermediate files:
+```
+mkdir -p test && cd test
+ulimit -n 1024
+NEvents=5 NEventsQED=100 $O2_ROOT/prodtests/full_system_test.sh
+```
+`ulimit -n 1024` avoids the workflow running out of file descriptors. `NEventsQED` controls the QED background, which dominates the TPC occupancy, so lower it for a smaller dataset.
+
+This leaves the raw time frames in `raw/` and already runs the reconstruction once. To get the dumps, re-run only the reconstruction over that raw data with the dump enabled:
 ```
-DISABLE_PROCESSING=1 NEvents=5 NEventsQED=100 SHMSIZE=16000000000 $O2_ROOT/prodtests/full_system_test.sh
+ARGS_ALL_EXTRA="--completion-policy=wait --exit-transition-timeout 20" \
+  CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow="GPU_global.dump=1;"      \
+  WORKFLOW_DETECTORS=TPC SYNCMODE=1 SHMSIZE=$((16<<30)) EXTINPUT=1    \
+  $O2_ROOT/prodtests/full-system-test/dpl-workflow.sh
 ```
-should be enough.
+`CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow="GPU_global.dump=1;"` is what writes the `.dump` files; everything else just shapes the workflow. `--completion-policy=wait` and `--exit-transition-timeout 20` let the workflow drain and shut down by itself once the input ends, instead of having to be killed. `EXTINPUT=1` takes the raw data from the `readout-proxy` FMQ channel, which means a feeder has to run in a second shell (`datadistribution.sh` from the same folder, which needs the `DataDistribution` package). Leave `EXTINPUT` unset to have `dpl-workflow.sh` read the raw files directly instead, which needs no second process. See [env-variables.md](https://github.com/AliceO2Group/AliceO2/blob/dev/prodtests/full-system-test/documentation/env-variables.md) for the other variables.
 
-Afterwards run the following command to dump the data:
+Finally collect the result into a dataset:
 ```
-SYNCMODE=1 CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow="GPU_global.dump=1;" WORKFLOW_DETECTORS=TPC SHMSIZE=16000000000 $O2_ROOT/prodtests/full-system-test/dpl-workflow.sh
+mkdir -p ~/standalone/events/fst-tpc && cp *.dump ~/standalone/events/fst-tpc/
 ```
 
 To dump standalone data from CTF raw data in `myctf.root`, you can use the same script, e.g.:
@@ -101,3 +125,14 @@ CTFINPUT=1 INPUT_FILE_LIST=myctf.root CONFIG_EXTRA_PROCESS_o2_gpu_reco_workflow=
 ```
 
 On the EPNs, you can find some reference data sets at `/home/drohr/standalone/events`.
+
+## Using the built-in event generator
+
+With `--eventGenerator` the benchmark generates simple synthetic events itself, without any simulation or raw data. The parent `events/` folder has to exist, since the generator only creates the dataset folder itself:
+```
+mkdir -p events
+./ca --eventGenerator -e gen --setMaxTimeBin 0 --rundEdx 0
+```
+`--setMaxTimeBin` has to be given, because the GRP settings are not read from a dataset in this mode and their default is the invalid sentinel `-2`.
+
+This writes `settings.dump` and the `event.<i>.dump` files, but no calibration objects, so the generated events can be inspected and used for the generator's own QA, but tracking on them stops at `Need cluster native data for on-the-fly TPC transform`. To track generated events, copy a `tpctransform.dump` from a dataset produced as described above into the folder.

dependencies/FindO2GPU.cmake

@@ -10,7 +10,7 @@
 # or submit itself to any jurisdiction.
 
 # NOTE!!!! - Whenever this file is changed, move it over to alidist/resources
-# FindO2GPU.cmake Version 23
+# FindO2GPU.cmake Version 24
 
 set(CUDA_COMPUTETARGET_DEFAULT_FULL 80-real;86-real;89-real;120-real;75-virtual)
 set(HIP_AMDGPUTARGET_DEFAULT_FULL gfx906;gfx908)
@@ -21,8 +21,7 @@ if(NOT DEFINED ENABLE_CUDA)
   set(ENABLE_CUDA "AUTO")
 endif()
 if(NOT DEFINED ENABLE_METAL)
-  # OFF rather than AUTO while the backend is unproven
-  set(ENABLE_METAL "OFF")
+  set(ENABLE_METAL "AUTO")
 endif()
 if(NOT DEFINED ENABLE_OPENCL)
   set(ENABLE_OPENCL "AUTO")