trino: Query fails with EXCEEDED_LOCAL_MEMORY_LIMIT on 474 when JVM set higher
SELECT kind, max(length(payload)) max_payload_len
FROM hive.default.test_payload_tbl
GROUP BY 1
fails with INSUFFICIENT_RESOURCES when JVM is set to 86GB on EC2 r6a.4xlarge
io.trino.ExceededMemoryLimitException: Query exceeded per-node memory limit of 25.80GB [Allocated: 25.79GB, Delta: 14.39MB, Top Consumers: {ScanFilterAndProjectOperator=25.78GB, HashAggregationOperator=7.92MB, PagePartitioner=4.08MB}]
at io.trino.ExceededMemoryLimitException.exceededLocalUserMemoryLimit(ExceededMemoryLimitException.java:40)
at io.trino.memory.QueryContext.enforceUserMemoryLimit(QueryContext.java:330)
at io.trino.memory.QueryContext.updateUserMemory(QueryContext.java:165)
at io.trino.memory.QueryContext.lambda$addTaskContext$0(QueryContext.java:250)
at io.trino.memory.QueryContext$QueryMemoryReservationHandler.reserveMemory(QueryContext.java:311)
at io.trino.memory.context.RootAggregatedMemoryContext.updateBytes(RootAggregatedMemoryContext.java:37)
at io.trino.memory.context.ChildAggregatedMemoryContext.updateBytes(ChildAggregatedMemoryContext.java:38)
at io.trino.memory.context.ChildAggregatedMemoryContext.updateBytes(ChildAggregatedMemoryContext.java:38)
at io.trino.memory.context.ChildAggregatedMemoryContext.updateBytes(ChildAggregatedMemoryContext.java:38)
at io.trino.memory.context.SimpleLocalMemoryContext.setBytes(SimpleLocalMemoryContext.java:65)
at io.trino.operator.OperatorContext$InternalLocalMemoryContext.setBytes(OperatorContext.java:701)
at io.trino.operator.ScanFilterAndProjectOperator$SplitToPages.lambda$processPageSource$4(ScanFilterAndProjectOperator.java:299)
at io.trino.operator.WorkProcessorUtils$BlockingProcess.process(WorkProcessorUtils.java:210)
at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:423)
at io.trino.operator.WorkProcessorUtils.lambda$flatten$6(WorkProcessorUtils.java:317)
at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:359)
at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:423)
at io.trino.operator.WorkProcessorUtils$3.process(WorkProcessorUtils.java:346)
at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:423)
at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:261)
at io.trino.operator.WorkProcessorUtils.lambda$processStateMonitor$2(WorkProcessorUtils.java:240)
at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:423)
at io.trino.operator.WorkProcessorUtils.getNextState(WorkProcessorUtils.java:261)
at io.trino.operator.WorkProcessorUtils.lambda$finishWhen$3(WorkProcessorUtils.java:255)
at io.trino.operator.WorkProcessorUtils$ProcessWorkProcessor.process(WorkProcessorUtils.java:423)
at io.trino.operator.WorkProcessorSourceOperatorAdapter.getOutput(WorkProcessorSourceOperatorAdapter.java:133)
at io.trino.operator.Driver.processInternal(Driver.java:403)
at io.trino.operator.Driver.lambda$process$8(Driver.java:306)
at io.trino.operator.Driver.tryWithLock(Driver.java:709)
at io.trino.operator.Driver.process(Driver.java:298)
at io.trino.operator.Driver.processForDuration(Driver.java:269)
at io.trino.execution.SqlTaskExecution$DriverSplitRunner.processFor(SqlTaskExecution.java:890)
at io.trino.execution.executor.dedicated.SplitProcessor.run(SplitProcessor.java:77)
at io.trino.execution.executor.dedicated.TaskEntry$VersionEmbedderBridge.lambda$run$0(TaskEntry.java:201)
at io.trino.$gen.Trino_474____20250331_090836_2.run(Unknown Source)
at io.trino.execution.executor.dedicated.TaskEntry$VersionEmbedderBridge.run(TaskEntry.java:202)
at io.trino.execution.executor.scheduler.FairScheduler.runTask(FairScheduler.java:177)
at io.trino.execution.executor.scheduler.FairScheduler.lambda$submit$0(FairScheduler.java:164)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:572)
at com.google.common.util.concurrent.TrustedListenableFutureTask$TrustedFutureInterruptibleTask.runInterruptibly(TrustedListenableFutureTask.java:131)
at com.google.common.util.concurrent.InterruptibleTask.run(InterruptibleTask.java:75)
at com.google.common.util.concurrent.TrustedListenableFutureTask.run(TrustedListenableFutureTask.java:82)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at java.base/java.lang.Thread.run(Thread.java:1575)
Same query finishes successfully when JVM is set to 9GB on EC2 r5.large
Both has spill-enabled=TRUE
show create table hive.default.test_payload_tbl;
CREATE TABLE hive.default.test_payload_tbl (
kind varchar(5),
payload varchar
)
WITH (
format = 'ORC'
)
About this issue
- Original URL
- State: closed
- Created 2 years ago
- Reactions: 1
- Comments: 20 (17 by maintainers)
I can approve the the PR solves the issue. Thanks!
@guyco33 can you check whether the PR that was merged fixes the issue?
I can approve that problem starts after the change done in SourcePage PR
I’ve tested the following snapshots:
Snapshot
472-10-GD2C414A- OKSnapshopt
472-29-GE437B16- Issue starts@dain FYI