prometheus-operator: can not reload prometheus rule

Your ruleSelector appears to be inside of securityContext. The securityContext block doesn’t look like it’s valid in that location either

I logged into the node where the pod is located. Then find the Prometheus process in the node, and use strace to trace the kill command. I found that the sighup signal didn’t reach the Prometheus process.

core@10 ~ $ ps -ef | grep prometheus | grep web.enable
1000      1103  1081 15 Oct15 ?        1-04:13:51 /bin/prometheus --web.console.templates=/etc/prometheus/consoles --web.console.libraries=/etc/prometheus/console_libraries --config.file=/etc/prometheus/config_out/prometheus.env.yaml --storage.tsdb.path=/prometheus --storage.tsdb.retention=60d --web.enable-lifecycle --storage.tsdb.no-lockfile --web.route-prefix=/

core@10 ~ $ strace kill -HUP 1103
execve("/usr/bin/kill", ["kill", "-HUP", "1103"], [/* 27 vars */]) = 0
brk(0)                                  = 0x55f000b56000
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fb01a59d000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=31667, ...}) = 0
mmap(NULL, 31667, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fb01a595000
close(3)                                = 0
open("/lib64/libprocps.so.3", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20B\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=75824, ...}) = 0
mmap(NULL, 2246760, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fb01a156000
mprotect(0x7fb01a167000, 2093056, PROT_NONE) = 0
mmap(0x7fb01a366000, 12288, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x10000) = 0x7fb01a366000
mmap(0x7fb01a369000, 71784, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fb01a369000
close(3)                                = 0
open("/lib64/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\6\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1836592, ...}) = 0
mmap(NULL, 3944032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fb019d93000
mprotect(0x7fb019f4d000, 2093056, PROT_NONE) = 0
mmap(0x7fb01a14c000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1b9000) = 0x7fb01a14c000
mmap(0x7fb01a152000, 15968, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fb01a152000
close(3)                                = 0
open("/lib64/libdl.so.2", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\0\16\0\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=14360, ...}) = 0
mmap(NULL, 2109584, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fb019b8f000
mprotect(0x7fb019b92000, 2093056, PROT_NONE) = 0
mmap(0x7fb019d91000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x2000) = 0x7fb019d91000
close(3)                                = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fb01a593000
arch_prctl(ARCH_SET_FS, 0x7fb01a593b40) = 0
mprotect(0x7fb01a14c000, 16384, PROT_READ) = 0
mprotect(0x7fb019d91000, 4096, PROT_READ) = 0
mprotect(0x7fb01a366000, 8192, PROT_READ) = 0
mprotect(0x55f000286000, 4096, PROT_READ) = 0
mprotect(0x7fb01a59f000, 4096, PROT_READ) = 0
munmap(0x7fb01a595000, 31667)           = 0
uname({sys="Linux", node="10.106.124.43", ...}) = 0
open("/sys/devices/system/cpu/online", O_RDONLY|O_CLOEXEC) = 3
read(3, "0-7\n", 8192)                  = 4
close(3)                                = 0
getpid()                                = 13545
kill(1103, SIGHUP)                      = -1 EPERM (Operation not permitted)
close(1)                                = 0
close(2)                                = 0
exit_group(1)                           = ?
+++ exited with 1 +++