tendermint: Tendermint stops building blocks

BUG REPORT

Tendermint version 0.20.0-27bd1dea

ABCI app Custom closed Source. Based on jABCI (https://github.com/jTendermint/jabci)

Environment:

• OSX
• Binary release package
• 4 nodes, 3 validators, all running localhost
• UPDATE: 1 Validator is entered as seed/persistent peer in all other nodes.
• create_empty_blocks == false

What happened: After several blocks, tendermint no longer creates new blocks on new transactions. CheckTx runs successful and a added good tx message is found in the logs but nothing else happens. There are several tx types. Most tx are around 10kb in size. The tx that is tried to be sent is ~280bytes Also, tendermint reports several older tx to be as duplicate in the mempool, no idea where that is coming from.

Please see the following dump consensus output of the validator node, to which the tx is sent:

{
  "jsonrpc": "2.0",
  "id": "",
  "result": {
    "round_state": {
      "height": 187,
      "round": 0,
      "step": 4,
      "start_time": "2018-06-15T17:11:48.984420411+02:00",
      "commit_time": "0001-01-01T00:00:00Z",
      "validators": {
        "validators": [
          {
            "address": "377218B1A5419768569B7829CE95C774F125DB35",
            "pub_key": {
              "type": "AC26791624DE60",
              "value": "8xEJkW3ghtHrLg0JBXvRi4xzUEy4KkuqMGtgODlkE/U="
            },
            "voting_power": 10,
            "accum": -20
          },
          {
            "address": "7F529D2F03B68924E5BB5F880C828CF8D134B8BD",
            "pub_key": {
              "type": "AC26791624DE60",
              "value": "eRmCyNdnKbsJWyFkSxLWcyduXKKdu/ZG2eue3ud3dA0="
            },
            "voting_power": 10,
            "accum": 10
          },
          {
            "address": "B5A8E53B84C94FB98C94C487FC19B0CC4F490DEA",
            "pub_key": {
              "type": "AC26791624DE60",
              "value": "xZYfYQGfrDi4dFF1ETY6qRGFfyZ6HUg7OpdRJpS1BAY="
            },
            "voting_power": 10,
            "accum": 10
          }
        ],
        "proposer": {
          "address": "377218B1A5419768569B7829CE95C774F125DB35",
          "pub_key": {
            "type": "AC26791624DE60",
            "value": "8xEJkW3ghtHrLg0JBXvRi4xzUEy4KkuqMGtgODlkE/U="
          },
          "voting_power": 10,
          "accum": -20
        }
      },
      "proposal": null,
      "proposal_block": null,
      "proposal_block_parts": null,
      "locked_round": 0,
      "locked_block": null,
      "locked_block_parts": null,
      "valid_round": 0,
      "valid_block": null,
      "valid_block_parts": null,
      "votes": [
        {
          "round": 0,
          "prevotes": [
            "nil-Vote",
            "nil-Vote",
            "Vote{2:B5A8E53B84C9 187/00/1(Prevote) 000000000000 /10109156264A.../ @ 2018-06-15T13:03:28.555Z}"
          ],
          "prevotes_bit_array": "BA{3:__x} 10/30 = 0.33",
          "precommits": [
            "nil-Vote",
            "nil-Vote",
            "nil-Vote"
          ],
          "precommits_bit_array": "BA{3:___} 0/30 = 0.00"
        }
      ],
      "commit_round": -1,
      "last_commit": {
        "votes": [
          "Vote{0:377218B1A541 186/00/2(Precommit) D59853BA0199 /4A91804010D3.../ @ 2018-06-15T13:03:25.447Z}",
          "Vote{1:7F529D2F03B6 186/00/2(Precommit) D59853BA0199 /E922055E2CE0.../ @ 2018-06-15T13:03:25.447Z}",
          "Vote{2:B5A8E53B84C9 186/00/2(Precommit) D59853BA0199 /E625746B6EBA.../ @ 2018-06-15T13:03:25.349Z}"
        ],
        "votes_bit_array": "BA{3:xxx} 30/30 = 1.00",
        "peer_maj_23s": {}
      },
      "last_validators": {
        "validators": [
          {
            "address": "377218B1A5419768569B7829CE95C774F125DB35",
            "pub_key": {
              "type": "AC26791624DE60",
              "value": "8xEJkW3ghtHrLg0JBXvRi4xzUEy4KkuqMGtgODlkE/U="
            },
            "voting_power": 10,
            "accum": 0
          },
          {
            "address": "7F529D2F03B68924E5BB5F880C828CF8D134B8BD",
            "pub_key": {
              "type": "AC26791624DE60",
              "value": "eRmCyNdnKbsJWyFkSxLWcyduXKKdu/ZG2eue3ud3dA0="
            },
            "voting_power": 10,
            "accum": 0
          },
          {
            "address": "B5A8E53B84C94FB98C94C487FC19B0CC4F490DEA",
            "pub_key": {
              "type": "AC26791624DE60",
              "value": "xZYfYQGfrDi4dFF1ETY6qRGFfyZ6HUg7OpdRJpS1BAY="
            },
            "voting_power": 10,
            "accum": 0
          }
        ],
        "proposer": {
          "address": "B5A8E53B84C94FB98C94C487FC19B0CC4F490DEA",
          "pub_key": {
            "type": "AC26791624DE60",
            "value": "xZYfYQGfrDi4dFF1ETY6qRGFfyZ6HUg7OpdRJpS1BAY="
          },
          "voting_power": 10,
          "accum": 0
        }
      }
    },
    "peers": [
      {
        "node_address": "b7ea63cf5733de753713fbf310bdbc208be75db9@192.168.55.1:46676",
        "peer_state": {
          "round_state": {
            "height": 187,
            "round": 0,
            "step": 2,
            "start_time": "2018-06-15T15:03:27.114897224+02:00",
            "proposal": false,
            "proposal_block_parts_header": {
              "total": 0,
              "hash": ""
            },
            "proposal_block_parts": null,
            "proposal_pol_round": -1,
            "proposal_pol": "___",
            "prevotes": "__x",
            "precommits": "___",
            "last_commit_round": 0,
            "last_commit": null,
            "catchup_commit_round": -1,
            "catchup_commit": "___"
          },
          "stats": {
            "last_vote_height": 0,
            "votes": 0,
            "last_block_part_height": 0,
            "block_parts": 0
          }
        }
      },
      {
        "node_address": "852b63f18af14884768aac6837b3643a6b3a9241@192.168.55.1:46666",
        "peer_state": {
          "round_state": {
            "height": 187,
            "round": 0,
            "step": 2,
            "start_time": "2018-06-15T15:03:27.118553416+02:00",
            "proposal": false,
            "proposal_block_parts_header": {
              "total": 0,
              "hash": ""
            },
            "proposal_block_parts": null,
            "proposal_pol_round": -1,
            "proposal_pol": "___",
            "prevotes": "__x",
            "precommits": "___",
            "last_commit_round": 0,
            "last_commit": null,
            "catchup_commit_round": -1,
            "catchup_commit": "___"
          },
          "stats": {
            "last_vote_height": 0,
            "votes": 0,
            "last_block_part_height": 0,
            "block_parts": 0
          }
        }
      },
      {
        "node_address": "83bfefedfe49668513e2067dbd6c6d2d158ba0a8@192.168.55.1:46686",
        "peer_state": {
          "round_state": {
            "height": 187,
            "round": 0,
            "step": 2,
            "start_time": "2018-06-15T15:03:26.784035233+02:00",
            "proposal": false,
            "proposal_block_parts_header": {
              "total": 0,
              "hash": ""
            },
            "proposal_block_parts": null,
            "proposal_pol_round": -1,
            "proposal_pol": "___",
            "prevotes": "__x",
            "precommits": "___",
            "last_commit_round": 0,
            "last_commit": "___",
            "catchup_commit_round": -1,
            "catchup_commit": "___"
          },
          "stats": {
            "last_vote_height": 187,
            "votes": 1,
            "last_block_part_height": 0,
            "block_parts": 0
          }
        }
      }
    ]
  }
}

What you expected to happen: Block should be built on new tx

Anything else do we need to know: Using max_block_size_txs = 100 due to the fact that the default 10k leads to problems with hardcoded blocksize limit in load test scenarios (when we really pump tx)