kubernetes: POD is OOM.Can't get the limited memory of the container
POD is OOM. k8s v1.19.4+docker is OK. but v1.21+cri-o+buildah does not work properly. Why the limits are not effective.Not the container limit memory, It’s physical memory
centos7
[root@master01:.docker_temp] # cat /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)
kernel
[root@master01:.docker_temp] # uname -r
5.4.126-1.el7.elrepo.x86_64
use limits
[root@master01:.docker_temp] # kubectl get pod system-timer-7f46c5c475-4d7bm -o yaml
spec:
containers:
- name: system-timer
imagePullPolicy: Always
resources:
limits:
cpu: "2"
memory: 2Gi
requests:
cpu: 500m
memory: 512Mi
securityContext:
privileged: false
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirstWithHostNet
enableServiceLinks: true
memory
[root@master01:.docker_temp] # kubectl exec -it system-timer-7f46c5c475-4d7bm -- free -h
total used free shared buff/cache available
Mem: 15Gi 9.8Gi 4.1Gi 22Mi 1.7Gi 5.7Gi
Swap: 0B 0B 0B
use -XX:+UseContainerSupport
[root@master01:.docker_temp] # kubectl exec -it system-timer-7f46c5c475-4d7bm -- ps auxwww
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 1 62.2 11.4 12132868 1872392 ? Ssl 20:22 2:18 java -server -jar -XX:+UseContainerSupport -Dlogging.level.root=INFO -Dapp.name=system-timer -Dspring.profiles.active=test /system-timer.jar
root 331 0.0 0.0 7636 2720 pts/0 Rs+ 20:25 0:00 ps auxwww
Cannot read memory.limit_in_bytes
[root@master01:.docker_temp] # kubectl exec -it system-timer-7f46c5c475-4d7bm -- java -XshowSettings:vm -version
VM settings:
Max. Heap Size (Estimated): 3.48G
Ergonomics Machine Class: server
Using VM: OpenJDK 64-Bit Server VM
openjdk version "1.8.0_302"
OpenJDK Runtime Environment (build 1.8.0_302-b08)
OpenJDK 64-Bit Server VM (build 25.302-b08, mixed mode)
k8s version
[root@master01:.docker_temp] # kubectl version
Client Version: version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.1", GitCommit:"5e58841cce77d4bc13713ad2b91fa0d961e69192", GitTreeState:"clean", BuildDate:"2021-05-12T14:18:45Z", GoVersion:"go1.16.4", Compiler:"gc", Platform:"linux/amd64"}
Server Version: version.Info{Major:"1", Minor:"21", GitVersion:"v1.21.1", GitCommit:"5e58841cce77d4bc13713ad2b91fa0d961e69192", GitTreeState:"clean", BuildDate:"2021-05-12T14:12:29Z", GoVersion:"go1.16.4", Compiler:"gc", Platform:"linux/amd64"}
cri-o version
[root@master01:.docker_temp] # crio --version
crio version 1.21.0
Version: 1.21.0
GitCommit: bc1ef35a932acc2f6f3b6d3eb19a4f68aa9423f6
GitTreeState: clean
BuildDate: 2021-04-30T13:46:48Z
GoVersion: go1.15.5
Compiler: gc
Platform: linux/amd64
Linkmode: dynamic
buildah version
[root@master01:.docker_temp] # buildah --version
buildah version 1.19.6 (image-spec 1.0.1-dev, runtime-spec 1.0.2-dev)
oracleJdk or openjdk have same problem
[root@master01:.docker_temp] # kubectl exec -it system-timer-7f46c5c475-4d7bm -- jinfo 1
Attaching to process ID 1, please wait...
Debugger attached successfully.
Server compiler detected.
JVM version is 25.302-b08
Java System Properties:
java.runtime.name = OpenJDK Runtime Environment
java.vm.version = 25.302-b08
sun.boot.library.path = /usr/local/openjdk-8/jre/lib/amd64
java.protocol.handler.pkgs = org.springframework.boot.loader
java.vendor.url = http://java.oracle.com/
java.vm.vendor = Oracle Corporation
path.separator = :
file.encoding.pkg = sun.io
java.vm.name = OpenJDK 64-Bit Server VM
sun.os.patch.level = unknown
sun.java.launcher = SUN_STANDARD
user.country = US
user.dir = /
java.vm.specification.name = Java Virtual Machine Specification
PID = 1
java.runtime.version = 1.8.0_302-b08
java.awt.graphicsenv = sun.awt.X11GraphicsEnvironment
JM.LOG.PATH = /root/logs
csp.sentinel.api.port = 8719
os.arch = amd64
java.endorsed.dirs = /usr/local/openjdk-8/jre/lib/endorsed
spring.profiles.active = test
line.separator =
java.io.tmpdir = /tmp
java.vm.specification.vendor = Oracle Corporation
os.name = Linux
csp.sentinel.charset = UTF-8
csp.sentinel.flow.cold.factor = 3
app.name = system-timer
sun.jnu.encoding = ANSI_X3.4-1968
java.library.path = /usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
spring.beaninfo.ignore = true
sun.nio.ch.bugLevel =
csp.sentinel.app.name = timer
java.specification.name = Java Platform API Specification
java.class.version = 52.0
sun.management.compiler = HotSpot 64-Bit Tiered Compilers
os.version = 5.4.126-1.el7.elrepo.x86_64
user.home = /root
user.timezone = Etc/UTC
java.awt.printerjob = sun.print.PSPrinterJob
file.encoding = ANSI_X3.4-1968
@appId = timer
java.specification.version = 1.8
logging.level.root = INFO
user.name = root
java.class.path = /system-timer.jar
java.vm.specification.version = 1.8
sun.arch.data.model = 64
sun.java.command = /system-timer.jar
java.home = /usr/local/openjdk-8/jre
user.language = en
java.specification.vendor = Oracle Corporation
awt.toolkit = sun.awt.X11.XToolkit
java.vm.info = mixed mode
java.version = 1.8.0_302
java.ext.dirs = /usr/local/openjdk-8/jre/lib/ext:/usr/java/packages/lib/ext
sun.boot.class.path = /usr/local/openjdk-8/jre/lib/resources.jar:/usr/local/openjdk-8/jre/lib/rt.jar:/usr/local/openjdk-8/jre/lib/sunrsasign.jar:/usr/local/openjdk-8/jre/lib/jsse.jar:/usr/local/openjdk-8/jre/lib/jce.jar:/usr/local/openjdk-8/jre/lib/charsets.jar:/usr/local/openjdk-8/jre/lib/jfr.jar:/usr/local/openjdk-8/jre/classes
csp.sentinel.dashboard.server = 172.16.66.7:18087
java.awt.headless = true
java.vendor = Oracle Corporation
file.separator = /
java.vendor.url.bug = http://bugreport.sun.com/bugreport/
sun.io.unicode.encoding = UnicodeLittle
sun.cpu.endian = little
sun.cpu.isalist =
VM Flags:
Non-default VM flags: -XX:CICompilerCount=4 -XX:InitialHeapSize=264241152 -XX:MaxHeapSize=4198498304 -XX:MaxNewSize=1399324672 -XX:MinHeapDeltaBytes=524288 -XX:NewSize=88080384 -XX:OldSize=176160768 -XX:+UseCompressedClassPointers -XX:+UseCompressedOops -XX:+UseContainerSupport -XX:+UseParallelGC
Command line: -XX:+UseContainerSupport -Dlogging.level.root=INFO -Dapp.name=system-timer -Dspring.profiles.active=test
It is normal to use xms and Xmx
About this issue
- Original URL
- State: closed
- Created 3 years ago
- Comments: 26 (17 by maintainers)
thanks for your patience @sbwfnhn