pinot: Servers can't consume messages due to permission denied error in the endpoint '/segmentConsumed' when ACL is enabled

While using the ACL feature, Pinot Servers aren’t able to consume messages from Kafka due to a permission denied error in the endpoint /segmentConsumed (more details in the log).

Ps: The Servers are able to consume a few events before the error starts, but after that no more events are consumed.

Cluster version

Apache Pinot version: 0.12.1

Cluster ACL configs:

Controller ACL conf:


# ACLs
# Enable ACL enforcement in the controller
controller.admin.access.control.factory.class=org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory

# Create users "admin" and "user". Keep in mind we're not enforcing any ACLs yet.
controller.admin.access.control.principals=admin,user
# Set the admin's password to "verysecret"
controller.admin.access.control.principals.admin.password=verysecret
# Set the user's password to "secret" and allow "READ" only
controller.admin.access.control.principals.user.password=secret
controller.admin.access.control.principals.user.permissions=READ
controller.admin.access.control.principals.user.tables=events,events_upsert_full,events_upsert_partial
# Enable the controller to fetch segments by providing the credentials as a token "Basic " + base64encode("admin:verysecret")
controller.segment.fetcher.auth.token="Basic YWRtaW46dmVyeXNlY3JldA=="

Server ACL conf:

# ACLs
# Enable the Server to fetch/upload segments by providing the credentials as a token "Basic " + base64encode("admin:verysecret")
pinot.server.segment.fetcher.auth.token="Basic YWRtaW46dmVyeXNlY3JldA=="
pinot.server.segment.uploader.auth.token="Basic YWRtaW46dmVyeXNlY3JldA=="
pinot.server.instance.auth.token="Basic YWRtaW46dmVyeXNlY3JldA=="

Broker ACL conf:

# Enable ACL enforcement in the Broker
# The factory class property is different for the broker
pinot.broker.access.control.class=org.apache.pinot.broker.broker.BasicAuthAccessControlFactory

# Create the users and password (must be exactly the same as the ones created for the controller)
pinot.broker.access.control.principals=admin,user
pinot.broker.access.control.principals.admin.password=verysecret
pinot.broker.access.control.principals.user.password=secret

# No need to set READ permissions here since broker requests are read-only

# This configuration option allow specification of usernames and passwords as well as optional ACL restrictions on a per-table table basis (access type is always READ)
pinot.broker.access.control.principals.user.tables=events,events_upsert_full,events_upsert_partial

Minion ACL conf:

# ACLs
segment.fetcher.auth.token="Basic YWRtaW46dmVyeXNlY3JldA=="
task.auth.token="Basic YWRtaW46dmVyeXNlY3JldA=="

Table config

{
    "tableName": "events_upsert_full",
    "tableType": "REALTIME",
    "segmentsConfig": {
      "timeColumnName": "ts",
      "schemaName": "events",
      "replication": "2",
      "replicasPerPartition": "3",
      "retentionTimeUnit": "HOURS",
      "retentionTimeValue": "2",
      "peerSegmentDownloadScheme": "http"
    },
    "tableIndexConfig": {
      "loadMode": "MMAP",
      "nullHandlingEnabled": true,
      "invertedIndexColumns": ["uuid", "name"],
      "streamConfigs": {
        "streamType": "kafka",
        "stream.kafka.topic.name": "events",
        "stream.kafka.broker.list": "broker0:9092,broker1:9092,broker2:9092",
        "stream.kafka.consumer.type": "lowlevel",
        "stream.kafka.consumer.prop.auto.offset.reset": "smallest",
        "stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
        "stream.kafka.decoder.class.name": "org.apache.pinot.plugin.inputformat.avro.confluent.KafkaConfluentSchemaRegistryAvroMessageDecoder",        
        "stream.kafka.decoder.prop.format": "AVRO",
        "stream.kafka.decoder.prop.schema.registry.rest.url": "http://schema-registry:8081",
        "realtime.segment.flush.threshold.rows": "5",
        "realtime.segment.flush.threshold.time": "2m",
        "realtime.segment.flush.threshold.segment.size": "5M"
      }
    },
    "ingestionConfig": {
        "batchIngestionConfig": {
          "segmentIngestionType": "APPEND",
          "segmentIngestionFrequency": "DAILY"
        },
        "transformConfigs": []
      },
    "tenants": {
        "broker": "DefaultTenant",
        "server": "one"
    },
    "metadata": {},
    "instanceAssignmentConfigMap": {
        "CONSUMING": {
          "tagPoolConfig": {
            "tag": "one_REALTIME",
            "poolBased": true,
            "numPools": 2
          },
          "replicaGroupPartitionConfig": {
            "replicaGroupBased": true,
            "numReplicaGroups": 2,
            "numInstancesPerReplicaGroup": 2
          }
        },
        "COMPLETED": {
            "tagPoolConfig": {
              "tag": "one_OFFLINE",
              "poolBased": true,
              "numPools": 2
            },
            "replicaGroupPartitionConfig": {
                "replicaGroupBased": true,
                "numReplicaGroups": 2,
                "numInstancesPerReplicaGroup": 2
              }
          }
      },
    "routing": {
        "instanceSelectorType": "strictReplicaGroup"
      },
      "upsertConfig": {
        "mode": "FULL",
        "hashFunction": "NONE",
        "deleteRecordColumn": "delete",
        "enableSnapshot": true

      },
      "task": {
        "taskTypeConfigsMap": {
          "MergeRollupTask": {
            "5m.mergeType": "concat",
            "5m.bucketTimePeriod": "5m",
            "5m.bufferTimePeriod": "1m"
          }
        }
      }
  }

Schema config

{
    "schemaName": "events",
    "primaryKeyColumns": [
        "name"
    ],
    "dimensionFieldSpecs": [
      {
        "name": "uuid",
        "dataType": "STRING"
      },
      {
        "name": "name",
        "dataType": "STRING",
        "defaultNullValue": ""
      },
      {
        "name": "delete",
        "dataType": "BOOLEAN",
        "defaultNullValue": "false"
      }
    ],
    "metricFieldSpecs": [
      {
        "name": "count",
        "dataType": "INT"
      }
    ],
    "dateTimeFieldSpecs": [{
      "name": "ts",
      "dataType": "TIMESTAMP",
      "format" : "1:MILLISECONDS:EPOCH",
      "granularity": "1:MILLISECONDS"
    }]
  }

Log error

2023-08-22 08:57:21 2023/08/22 11:57:21.716 ERROR [ServerSegmentCompletionProtocolHandler] [events_upsert_partial__1__0__20230822T1152Z] Could not send request http://pinot-controller-0:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=5&instance=Server_pinot-server-0_8098&offset=-1&name=events_upsert_partial__1__0__20230822T1152Z&rowCount=5&memoryUsedBytes=1368
2023-08-22 08:57:21 org.apache.pinot.common.exception.HttpErrorStatusException: Got error status code: 403 (Forbidden) with reason: "Permission is denied for READ '/segmentConsumed'" while sending request: http://pinot-controller-0:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=5&instance=Server_pinot-server-0_8098&offset=-1&name=events_upsert_partial__1__0__20230822T1152Z&rowCount=5&memoryUsedBytes=1368 to controller: pinot-controller-0, version: Unknown
2023-08-22 08:57:21     at org.apache.pinot.common.utils.http.HttpClient.wrapAndThrowHttpException(HttpClient.java:442) ~[pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:21     at org.apache.pinot.common.utils.FileUploadDownloadClient.sendSegmentCompletionProtocolRequest(FileUploadDownloadClient.java:1039) ~[pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:21     at org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler.sendRequest(ServerSegmentCompletionProtocolHandler.java:217) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:21     at org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler.segmentConsumed(ServerSegmentCompletionProtocolHandler.java:184) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:21     at org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager.postSegmentConsumedMsg(LLRealtimeSegmentDataManager.java:1110) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:21     at org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager$PartitionConsumer.run(LLRealtimeSegmentDataManager.java:650) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:21     at java.lang.Thread.run(Thread.java:829) [?:?]
2023-08-22 08:57:21 2023/08/22 11:57:21.717 ERROR [LLRealtimeSegmentDataManager_events_upsert_partial__1__0__20230822T1152Z] [events_upsert_partial__1__0__20230822T1152Z] Holding after response from Controller: {"offset":-1,"status":"NOT_SENT","streamPartitionMsgOffset":null,"isSplitCommitType":false,"buildTimeSec":-1}
2023-08-22 08:57:22 2023/08/22 11:57:22.482 ERROR [ServerSegmentCompletionProtocolHandler] [events_upsert_full__1__0__20230822T1152Z] Could not send request http://pinot-controller-0:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=5&instance=Server_pinot-server-0_8098&offset=-1&name=events_upsert_full__1__0__20230822T1152Z&rowCount=5&memoryUsedBytes=1368
2023-08-22 08:57:22 org.apache.pinot.common.exception.HttpErrorStatusException: Got error status code: 403 (Forbidden) with reason: "Permission is denied for READ '/segmentConsumed'" while sending request: http://pinot-controller-0:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=5&instance=Server_pinot-server-0_8098&offset=-1&name=events_upsert_full__1__0__20230822T1152Z&rowCount=5&memoryUsedBytes=1368 to controller: pinot-controller-0, version: Unknown
2023-08-22 08:57:22     at org.apache.pinot.common.utils.http.HttpClient.wrapAndThrowHttpException(HttpClient.java:442) ~[pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:22     at org.apache.pinot.common.utils.FileUploadDownloadClient.sendSegmentCompletionProtocolRequest(FileUploadDownloadClient.java:1039) ~[pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:22     at org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler.sendRequest(ServerSegmentCompletionProtocolHandler.java:217) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:22     at org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler.segmentConsumed(ServerSegmentCompletionProtocolHandler.java:184) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:22     at org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager.postSegmentConsumedMsg(LLRealtimeSegmentDataManager.java:1110) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:22     at org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager$PartitionConsumer.run(LLRealtimeSegmentDataManager.java:650) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:22     at java.lang.Thread.run(Thread.java:829) [?:?]
2023-08-22 08:57:22 2023/08/22 11:57:22.484 ERROR [LLRealtimeSegmentDataManager_events_upsert_full__1__0__20230822T1152Z] [events_upsert_full__1__0__20230822T1152Z] Holding after response from Controller: {"offset":-1,"status":"NOT_SENT","streamPartitionMsgOffset":null,"isSplitCommitType":false,"buildTimeSec":-1}
2023-08-22 08:57:24 2023/08/22 11:57:24.732 ERROR [ServerSegmentCompletionProtocolHandler] [events_upsert_partial__1__0__20230822T1152Z] Could not send request http://pinot-controller-0:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=5&instance=Server_pinot-server-0_8098&offset=-1&name=events_upsert_partial__1__0__20230822T1152Z&rowCount=5&memoryUsedBytes=1368
2023-08-22 08:57:24 org.apache.pinot.common.exception.HttpErrorStatusException: Got error status code: 403 (Forbidden) with reason: "Permission is denied for READ '/segmentConsumed'" while sending request: http://pinot-controller-0:9000/segmentConsumed?reason=rowLimit&streamPartitionMsgOffset=5&instance=Server_pinot-server-0_8098&offset=-1&name=events_upsert_partial__1__0__20230822T1152Z&rowCount=5&memoryUsedBytes=1368 to controller: pinot-controller-0, version: Unknown
2023-08-22 08:57:24     at org.apache.pinot.common.utils.http.HttpClient.wrapAndThrowHttpException(HttpClient.java:442) ~[pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:24     at org.apache.pinot.common.utils.FileUploadDownloadClient.sendSegmentCompletionProtocolRequest(FileUploadDownloadClient.java:1039) ~[pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:24     at org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler.sendRequest(ServerSegmentCompletionProtocolHandler.java:217) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:24     at org.apache.pinot.server.realtime.ServerSegmentCompletionProtocolHandler.segmentConsumed(ServerSegmentCompletionProtocolHandler.java:184) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:24     at org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager.postSegmentConsumedMsg(LLRealtimeSegmentDataManager.java:1110) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:24     at org.apache.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager$PartitionConsumer.run(LLRealtimeSegmentDataManager.java:650) [pinot-all-0.12.1-jar-with-dependencies.jar:0.12.1-6e235a4ec2a16006337da04e118a435b5bb8f6d8]
2023-08-22 08:57:24     at java.lang.Thread.run(Thread.java:829) [?:?]

Server Appconfig

curl -X GET "http://localhost:38097/appconfigs" -H "accept: application/json" -H "Authorization: Basic YWRtaW46dmVyeXNlY3JldA=="
{
  "systemConfig" : {
    "arch" : "amd64",
    "name" : "Linux",
    "version" : "5.15.49-linuxkit-pr",
    "availableProcessors" : 4,
    "totalPhysicalMemory" : "7 GB",
    "freePhysicalMemory" : "164 MB",
    "totalSwapSpace" : "3 GB",
    "freeSwapSpace" : "2 GB"
  },
  "runtimeConfig" : {
    "numTotalThreads" : 69,
    "numCurrentThreads" : 52,
    "maxHeapSize" : "1 GB",
    "currentHeapSize" : "198 MB"
  },
  "pinotConfig" : {
    "pinot.server.instance.enable.split.commit" : "true",
    "pinot.server.storage.factory.class.s3" : "org.apache.pinot.plugin.filesystem.S3PinotFS",
    "pinot.server.segment.fetcher.s3.class" : "org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher",
    "pinot.server.storage.factory.s3.region" : "us-east-1",
    "pinot.server.storage.factory.s3.disableacl" : "false",
    "pinot.server.netty.port" : "8098",
    "pinot.cluster.name" : "pinot-s3-example",
    "pinot.server.instance.segment.store.uri" : "s3://pinot/deep-storage",
    "pinot.set.instance.id.to.hostname" : "true",
    "pinot.server.storage.factory.s3.accesskey" : "minio",
    "pinot.server.storage.factory.s3.endpoint" : "http://minio:9000",
    "pinot.server.segment.fetcher.protocols" : "file,http,s3",
    "pinot.server.instance.datadir" : "/var/pinot/server/data/index",
    "pinot.zk.server" : "zookeeper:2181",
    "pinot.server.adminapi.port" : "8097",
    "pinot.server.segment.uploader.auth.token" : "*****",
    "pinot.server.instance.id" : "Server_pinot-server-1_8098",
    "pinot.server.instance.auth.token" : "*****",
    "pinot.server.segment.fetcher.auth.token" : "*****",
    "pinot.server.instance.segmenttardir" : "/var/pinot/server/data/segment",
    "pinot.server.storage.factory.s3.secretkey" : "*****",
    "pinot.service.role" : "SERVER"
  },
  "jvmConfig" : {
    "args" : [ "-Dplugins.dir=/opt/pinot/plugins", "-Xms256M", "-Xmx1G", "-XX:+UseG1GC", "-XX:MaxGCPauseMillis=200", "-Xlog:gc:gc-pinot-server-1.log", "-Dplugins.dir=/opt/pinot/plugins", "-Dapp.name=pinot-admin", "-Dapp.pid=1", "-Dapp.repo=/opt/pinot/lib", "-Dapp.home=/opt/pinot", "-Dbasedir=/opt/pinot" ],
    "garbageCollectors" : [ "G1 Young Generation", "G1 Old Generation" ],
    "envVariables" : {
      "PATH" : "/usr/local/openjdk-11/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
      "HOSTNAME" : "pinot-server-1",
      "PINOT_HOME" : "/opt/pinot",
      "JAVA_HOME" : "/usr/local/openjdk-11",
      "JAVA_OPTS" : "-Dplugins.dir=/opt/pinot/plugins -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc:gc-pinot-server-1.log",
      "PWD" : "/opt/pinot",
      "JAVA_VERSION" : "11.0.16",
      "LANG" : "C.UTF-8",
      "HOME" : "/root"
    },
    "libraryPath" : "/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib",
    "bootClassPath" : null,
    "systemProperties" : {
      "awt.toolkit" : "sun.awt.X11.XToolkit",
      "jdk.tls.rejectClientInitiatedRenegotiation" : "true",
      "helixmanager.flappingTimeWindow" : "1",
      "java.specification.version" : "11",
      "sun.cpu.isalist" : "",
      "sun.jnu.encoding" : "UTF-8",
      "java.class.path" : "/opt/pinot/lib/pinot-all-0.12.1-jar-with-dependencies.jar:/opt/pinot/plugins/pinot-input-format/pinot-thrift/pinot-thrift-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-confluent-avro/pinot-confluent-avro-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-parquet/pinot-parquet-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-json/pinot-json-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-protobuf/pinot-protobuf-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-orc/pinot-orc-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-csv/pinot-csv-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-avro/pinot-avro-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-segment-uploader/pinot-segment-uploader-default/pinot-segment-uploader-default-0.12.1.jar:/opt/pinot/plugins/pinot-file-system/pinot-hdfs/pinot-hdfs-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-file-system/pinot-gcs/pinot-gcs-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-file-system/pinot-s3/pinot-s3-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-file-system/pinot-adls/pinot-adls-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-segment-writer/pinot-segment-writer-file-based/pinot-segment-writer-file-based-0.12.1.jar:/opt/pinot/plugins/pinot-environment/pinot-azure/pinot-azure-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-metrics/pinot-yammer/pinot-yammer-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-metrics/pinot-dropwizard/pinot-dropwizard-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/pinot-minion-builtin-tasks-0.12.1.jar:/opt/pinot/plugins/pinot-stream-ingestion/pinot-kinesis/pinot-kinesis-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-stream-ingestion/pinot-kafka-2.0/pinot-kafka-2.0-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pinot-batch-ingestion-standalone-0.12.1-shaded.jar",
      "java.vm.vendor" : "Oracle Corporation",
      "sun.arch.data.model" : "64",
      "java.vendor.url" : "https://openjdk.java.net/",
      "user.timezone" : "Etc/UTC",
      "os.name" : "Linux",
      "java.vm.specification.version" : "11",
      "sun.java.launcher" : "SUN_STANDARD",
      "sun.boot.library.path" : "/usr/local/openjdk-11/lib",
      "sun.java.command" : "org.apache.pinot.tools.admin.PinotAdministrator StartServer -zkAddress zookeeper:2181 -clusterName pinot-s3-example -configFileName /opt/pinot/conf/server.conf",
      "app.pid" : "1",
      "jdk.debug" : "release",
      "sun.cpu.endian" : "little",
      "user.home" : "/root",
      "user.language" : "en",
      "java.specification.vendor" : "Oracle Corporation",
      "java.version.date" : "2022-07-19",
      "java.home" : "/usr/local/openjdk-11",
      "file.separator" : "/",
      "basedir" : "/opt/pinot",
      "java.vm.compressedOopsMode" : "32-bit",
      "line.separator" : "\n",
      "java.specification.name" : "Java Platform API Specification",
      "java.vm.specification.vendor" : "Oracle Corporation",
      "java.awt.graphicsenv" : "sun.awt.X11GraphicsEnvironment",
      "sun.management.compiler" : "HotSpot 64-Bit Tiered Compilers",
      "plugins.dir" : "/opt/pinot/plugins",
      "java.runtime.version" : "11.0.16+8",
      "user.name" : "root",
      "path.separator" : ":",
      "os.version" : "5.15.49-linuxkit-pr",
      "java.runtime.name" : "OpenJDK Runtime Environment",
      "file.encoding" : "UTF-8",
      "java.vm.name" : "OpenJDK 64-Bit Server VM",
      "java.vendor.version" : "18.9",
      "app.name" : "pinot-admin",
      "java.vendor.url.bug" : "https://bugreport.java.com/bugreport/",
      "java.io.tmpdir" : "/tmp",
      "java.version" : "11.0.16",
      "user.dir" : "/opt/pinot",
      "os.arch" : "amd64",
      "java.vm.specification.name" : "Java Virtual Machine Specification",
      "java.awt.printerjob" : "sun.print.PSPrinterJob",
      "sun.os.patch.level" : "unknown",
      "app.home" : "/opt/pinot",
      "java.library.path" : "/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib",
      "java.vm.info" : "mixed mode, sharing",
      "java.vendor" : "Oracle Corporation",
      "java.vm.version" : "11.0.16+8",
      "app.repo" : "/opt/pinot/lib",
      "sun.io.unicode.encoding" : "UnicodeLittle",
      "java.class.version" : "55.0"
    }
  }

Controller appconf

 curl -X GET "http://localhost:9000/appconfigs" -H "accept: application/json" -H "Authorization: Basic YWRtaW46dmVyeXNlY3JldA=="
{
  "systemConfig" : {
    "arch" : "amd64",
    "name" : "Linux",
    "version" : "5.15.49-linuxkit-pr",
    "availableProcessors" : 4,
    "totalPhysicalMemory" : "7 GB",
    "freePhysicalMemory" : "163 MB",
    "totalSwapSpace" : "3 GB",
    "freeSwapSpace" : "1 GB"
  },
  "runtimeConfig" : {
    "numTotalThreads" : 203,
    "numCurrentThreads" : 125,
    "maxHeapSize" : "1 GB",
    "currentHeapSize" : "247 MB"
  },
  "pinotConfig" : {
    "controller.task.frequencyperiod" : "5m",
    "controller.data.dir" : "s3://pinot/deep-storage",
    "controller.enable.split.commit" : "true",
    "pinot.controller.segment.fetcher.s3.class" : "org.apache.pinot.common.utils.fetcher.PinotFSSegmentFetcher",
    "pinot.controller.storage.factory.s3.disableacl" : "false",
    "controller.admin.access.control.principals.user.tables" : "events,events_upsert_full,events_upsert_partial",
    "controller.allow.hlc.tables" : "false",
    "pinot.cluster.name" : "pinot-s3-example",
    "controller.task.scheduler.enabled" : "true",
    "pinot.set.instance.id.to.hostname" : "true",
    "controller.host" : "pinot-controller-0",
    "pinot.controller.storage.factory.s3.endpoint" : "http://minio:9000",
    "controller.retention.frequencyperiod" : "30m",
    "pinot.zk.server" : "zookeeper:2181",
    "controller.helix.cluster.name" : "pinot-s3-example",
    "controller.admin.access.control.principals" : "admin,user",
    "pinot.controller.storage.factory.s3.accesskey" : "minio",
    "controller.zk.str" : "zookeeper:2181",
    "pinot.service.role" : "CONTROLLER",
    "controller.admin.access.control.principals.admin.password" : "*****",
    "controller.admin.access.control.factory.class" : "org.apache.pinot.controller.api.access.BasicAuthAccessControlFactory",
    "controller.access.protocols.http.port" : "9000",
    "controller.local.temp.dir" : "/tmp/pinot-tmp-data/",
    "controller.admin.access.control.principals.user.password" : "*****",
    "pinot.controller.segment.fetcher.auth.token" : "*****",
    "controller.port" : "9000",
    "pinot.controller.storage.factory.s3.secretkey" : "*****",
    "pinot.controller.storage.factory.class.s3" : "org.apache.pinot.plugin.filesystem.S3PinotFS",
    "pinot.controller.segment.fetcher.protocols" : "file,http,s3",
    "pinot.controller.storage.factory.s3.region" : "us-east-1",
    "controller.admin.access.control.principals.user.permissions" : "READ",
    "controller.realtime.segment.deepstoreuploadretryenabled" : "true"
  },
  "jvmConfig" : {
    "args" : [ "-Dplugins.dir=/opt/pinot/plugins", "-Xms256M", "-Xmx1G", "-XX:+UseG1GC", "-XX:MaxGCPauseMillis=200", "-Xlog:gc:gc-pinot-controller-0.log", "-Dplugins.dir=/opt/pinot/plugins", "-Dapp.name=pinot-admin", "-Dapp.pid=1", "-Dapp.repo=/opt/pinot/lib", "-Dapp.home=/opt/pinot", "-Dbasedir=/opt/pinot" ],
    "garbageCollectors" : [ "G1 Young Generation", "G1 Old Generation" ],
    "envVariables" : {
      "PATH" : "/usr/local/openjdk-11/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
      "HOSTNAME" : "pinot-controller-0",
      "PINOT_HOME" : "/opt/pinot",
      "JAVA_HOME" : "/usr/local/openjdk-11",
      "JAVA_OPTS" : "-Dplugins.dir=/opt/pinot/plugins -Xms256M -Xmx1G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xlog:gc:gc-pinot-controller-0.log",
      "PWD" : "/opt/pinot",
      "JAVA_VERSION" : "11.0.16",
      "LANG" : "C.UTF-8",
      "HOME" : "/root"
    },
    "libraryPath" : "/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib",
    "bootClassPath" : null,
    "systemProperties" : {
      "awt.toolkit" : "sun.awt.X11.XToolkit",
      "jdk.tls.rejectClientInitiatedRenegotiation" : "true",
      "helixmanager.flappingTimeWindow" : "1",
      "java.specification.version" : "11",
      "sun.cpu.isalist" : "",
      "sun.jnu.encoding" : "UTF-8",
      "java.class.path" : "/opt/pinot/lib/pinot-all-0.12.1-jar-with-dependencies.jar:/opt/pinot/plugins/pinot-input-format/pinot-thrift/pinot-thrift-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-confluent-avro/pinot-confluent-avro-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-parquet/pinot-parquet-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-json/pinot-json-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-protobuf/pinot-protobuf-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-orc/pinot-orc-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-csv/pinot-csv-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-input-format/pinot-avro/pinot-avro-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-segment-uploader/pinot-segment-uploader-default/pinot-segment-uploader-default-0.12.1.jar:/opt/pinot/plugins/pinot-file-system/pinot-hdfs/pinot-hdfs-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-file-system/pinot-gcs/pinot-gcs-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-file-system/pinot-s3/pinot-s3-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-file-system/pinot-adls/pinot-adls-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-segment-writer/pinot-segment-writer-file-based/pinot-segment-writer-file-based-0.12.1.jar:/opt/pinot/plugins/pinot-environment/pinot-azure/pinot-azure-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-metrics/pinot-yammer/pinot-yammer-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-metrics/pinot-dropwizard/pinot-dropwizard-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-minion-tasks/pinot-minion-builtin-tasks/pinot-minion-builtin-tasks-0.12.1.jar:/opt/pinot/plugins/pinot-stream-ingestion/pinot-kinesis/pinot-kinesis-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-stream-ingestion/pinot-kafka-2.0/pinot-kafka-2.0-0.12.1-shaded.jar:/opt/pinot/plugins/pinot-batch-ingestion/pinot-batch-ingestion-standalone/pinot-batch-ingestion-standalone-0.12.1-shaded.jar",
      "java.vm.vendor" : "Oracle Corporation",
      "sun.arch.data.model" : "64",
      "java.vendor.url" : "https://openjdk.java.net/",
      "user.timezone" : "Etc/UTC",
      "os.name" : "Linux",
      "java.vm.specification.version" : "11",
      "sun.java.launcher" : "SUN_STANDARD",
      "sun.boot.library.path" : "/usr/local/openjdk-11/lib",
      "sun.java.command" : "org.apache.pinot.tools.admin.PinotAdministrator StartController -zkAddress zookeeper:2181 -configFileName /opt/pinot/conf/controller.conf",
      "app.pid" : "1",
      "jdk.debug" : "release",
      "sun.cpu.endian" : "little",
      "user.home" : "/root",
      "user.language" : "en",
      "java.specification.vendor" : "Oracle Corporation",
      "java.version.date" : "2022-07-19",
      "java.home" : "/usr/local/openjdk-11",
      "file.separator" : "/",
      "basedir" : "/opt/pinot",
      "java.vm.compressedOopsMode" : "32-bit",
      "line.separator" : "\n",
      "java.specification.name" : "Java Platform API Specification",
      "java.vm.specification.vendor" : "Oracle Corporation",
      "java.awt.graphicsenv" : "sun.awt.X11GraphicsEnvironment",
      "sun.management.compiler" : "HotSpot 64-Bit Tiered Compilers",
      "plugins.dir" : "/opt/pinot/plugins",
      "java.runtime.version" : "11.0.16+8",
      "user.name" : "root",
      "path.separator" : ":",
      "os.version" : "5.15.49-linuxkit-pr",
      "java.runtime.name" : "OpenJDK Runtime Environment",
      "file.encoding" : "UTF-8",
      "java.vm.name" : "OpenJDK 64-Bit Server VM",
      "java.vendor.version" : "18.9",
      "app.name" : "pinot-admin",
      "java.vendor.url.bug" : "https://bugreport.java.com/bugreport/",
      "java.io.tmpdir" : "/tmp",
      "java.version" : "11.0.16",
      "user.dir" : "/opt/pinot",
      "os.arch" : "amd64",
      "java.vm.specification.name" : "Java Virtual Machine Specification",
      "java.awt.printerjob" : "sun.print.PSPrinterJob",
      "sun.os.patch.level" : "unknown",
      "app.home" : "/opt/pinot",
      "java.library.path" : "/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib",
      "java.vm.info" : "mixed mode, sharing",
      "java.vendor" : "Oracle Corporation",
      "java.vm.version" : "11.0.16+8",
      "app.repo" : "/opt/pinot/lib",
      "sun.io.unicode.encoding" : "UnicodeLittle",
      "java.class.version" : "55.0"
    }
  }
}

Thank you very much!

About this issue

  • Original URL
  • State: open
  • Created 10 months ago
  • Reactions: 1
  • Comments: 21 (11 by maintainers)

Most upvoted comments

cc @apucher @xiangfu0 @snleee

This is also a good question to be posted in the slack troubleshooting channel

+1
Jackie-Jiang on Aug 23, 2023
Read more comments on GitHub
← pinot: Mulitstage queries doesn't run locally
plc4x: [Bug]: plc4j-tools-connection-cache: broken connections remaing in the cache on timeout →