Always set CPU affinity for benchmark jobs

Because metrics like compilation time requires CPU

Signed-off-by: Huy Do <huydhn@gmail.com>

.ci/pytorch/test.sh

@@ -595,7 +595,6 @@ test_perf_for_dashboard() {
     elif [[ "${TEST_CONFIG}" == *cpu_aarch64* ]]; then
       device=cpu_aarch64
     fi
-    test_inductor_set_cpu_affinity
   elif [[ "${TEST_CONFIG}" == *cuda_a10g* ]]; then
     device=cuda_a10g
   elif [[ "${TEST_CONFIG}" == *h100* ]]; then
@@ -604,6 +603,9 @@ test_perf_for_dashboard() {
     device=rocm
   fi
 
+  # Always set CPU affinity because metrics like compilation time requires CPU
+  test_inductor_set_cpu_affinity
+
   for mode in "${modes[@]}"; do
     if [[ "$mode" == "inference" ]]; then
       dtype=bfloat16