ClickHouse: Bad size of marks file (versions 19.13.1.11 and 19.13.2)

2019.08.20 10:56:30.062278 [ 77 ] {e781c38b-55fb-4048-995c-804c0cd24544} <Error> executeQuery: Code: 246, e.displayText() = DB::Exception: Bad size of marks file ‘/CDATA/clickhouse/data/OPS_DM/TRANSACTIONS_TEST_MT/201901_1_149_3/skp_idx_i1.mrk2’: 4488, must be: 23136 (version 19.13.1.11 (official build)) (from 127.0.0.1:42808) (in query: SELECT TRAN_MONTH, uniqExact(CARD) AS cards, count() AS trans, sum(TRAN_AMOUNT_RUB) / 100 AS sum FROM OPS_DM.TRANSACTIONS_TEST_MT WHERE (MCC = 4511) AND (MERCHANT_NAME LIKE ‘%blahblahblah%’) AND ((PART_KEY >= 20190101) AND (PART_KEY <= 20190631)) AND (IS_APPROVED = 1) AND (PS_ID = 1) GROUP BY TRAN_MONTH), Stack trace:

  1. /usr/bin/clickhouse-server(StackTrace::StackTrace()+0x30) [0x7f58f30]
  2. /usr/bin/clickhouse-server(DB::Exception::Exception(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, int)+0x25) [0x3c427b5]
  3. /usr/bin/clickhouse-server() [0x39d0377]
  4. /usr/bin/clickhouse-server(DB::MergeTreeReaderStream::loadMarks()+0x5aa) [0x70a6c9a]
  5. /usr/bin/clickhouse-server(DB::MergeTreeReaderStream::MergeTreeReaderStream(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, unsigned long, std::vector<DB::MarkRange, std::allocatorDB::MarkRange > const&, DB::MarkCache*, bool, DB::UncompressedCache*, unsigned long, unsigned long, unsigned long, DB::MergeTreeIndexGranularityInfo const*, std::function<void (DB::ReadBufferFromFileBase::ProfileInfo)> const&, int)+0x498) [0x70a80c8]
  6. /usr/bin/clickhouse-server(DB::MergeTreeIndexReader::MergeTreeIndexReader(std::shared_ptr<DB::IMergeTreeIndex const>, std::shared_ptr<DB::MergeTreeDataPart const>, unsigned long, std::vector<DB::MarkRange, std::allocatorDB::MarkRange > const&)+0x347) [0x7096927]
  7. /usr/bin/clickhouse-server(DB::MergeTreeDataSelectExecutor::filterMarksUsingIndex(std::shared_ptr<DB::IMergeTreeIndex const>, std::shared_ptrDB::IMergeTreeIndexCondition, std::shared_ptr<DB::MergeTreeDataPart const>, std::vector<DB::MarkRange, std::allocatorDB::MarkRange > const&, DB::Settings const&) const+0x2d6) [0x70842d6]
  8. /usr/bin/clickhouse-server(DB::MergeTreeDataSelectExecutor::readFromParts(std::vector<std::shared_ptr<DB::MergeTreeDataPart const>, std::allocator<std::shared_ptr<DB::MergeTreeDataPart const> > >, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, DB::SelectQueryInfo const&, DB::Context const&, unsigned long, unsigned int, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, long, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, long> > > const*) const+0x1569) [0x70862f9]
  9. /usr/bin/clickhouse-server(DB::MergeTreeDataSelectExecutor::read(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, DB::SelectQueryInfo const&, DB::Context const&, unsigned long, unsigned int, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, long, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, long> > > const*) const+0x79) [0x7089159]
  10. /usr/bin/clickhouse-server(DB::StorageMergeTree::read(std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, DB::SelectQueryInfo const&, DB::Context const&, DB::QueryProcessingStage::Enum, unsigned long, unsigned int)+0x32) [0x6f885c2]
  11. /usr/bin/clickhouse-server(void DB::InterpreterSelectQuery::executeFetchColumnsDB::InterpreterSelectQuery::Pipeline(DB::QueryProcessingStage::Enum, DB::InterpreterSelectQuery::Pipeline&, std::shared_ptrDB::PrewhereInfo const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&)+0x1b6d) [0x6d712ed]
  12. /usr/bin/clickhouse-server(void DB::InterpreterSelectQuery::executeImplDB::InterpreterSelectQuery::Pipeline(DB::InterpreterSelectQuery::Pipeline&, std::shared_ptrDB::IBlockInputStream const&, bool)+0x668) [0x6d77768]
  13. /usr/bin/clickhouse-server(DB::InterpreterSelectQuery::executeWithMultipleStreams()+0x51) [0x6d5cf51]
  14. /usr/bin/clickhouse-server(DB::InterpreterSelectWithUnionQuery::executeWithMultipleStreams()+0x63) [0x6d79163]
  15. /usr/bin/clickhouse-server(DB::InterpreterSelectWithUnionQuery::execute()+0x38) [0x6d7b028]
  16. /usr/bin/clickhouse-server() [0x6e874db]
  17. /usr/bin/clickhouse-server(DB::executeQuery(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, DB::Context&, bool, DB::QueryProcessingStage::Enum, bool)+0xfc) [0x6e885dc]
  18. /usr/bin/clickhouse-server(DB::TCPHandler::runImpl()+0x9fa) [0x3c83f0a]
  19. /usr/bin/clickhouse-server(DB::TCPHandler::run()+0x2b) [0x3c847cb]
  20. /usr/bin/clickhouse-server(Poco::Net::TCPServerConnection::start()+0x10) [0x792cde0]
  21. /usr/bin/clickhouse-server(Poco::Net::TCPServerDispatcher::run()+0xed) [0x792d4fd]
  22. /usr/bin/clickhouse-server(Poco::PooledThread::run()+0x81) [0x8057e21]
  23. /usr/bin/clickhouse-server(Poco::ThreadImpl::runnableEntry(void*)+0x3c) [0x8055bcc]
  24. /usr/bin/clickhouse-server() [0xba2baa0]
  25. /lib64/libpthread.so.0(+0x7e25) [0x7f0f2f703e25]
  26. /lib64/libc.so.6(clone+0x6d) [0x7f0f2ef27bad]

Table structure: (RECORD_ID UInt64, PART_KEY UInt32, SPEC String, PROC_DATE Date, TRAN_DATE Date, TRAN_WEEK Date, TRAN_MONTH Date, TRAN_QUARTER Date, TRAN_WEEKDAY UInt8, TRAN_YEARMONTH UInt32, TRAN_DATETIME DateTime, LOCAL_DATETIME DateTime, GMT_DATETIME DateTime, MTI Int16, PS_ID Int8, CARD UInt64, CARD_MASK String, CARD_TRUNCATED String, CARD_LAST_DIGITS UInt16, CARD_TOKEN UInt64, CARD_LENGTH UInt8, CARD_SHA256 String, CARD_CITYHASH64 UInt64, CARD_SERVICE_CODE Int16, CARD_SEQUENCE Int16, CARD_PRODUCT String, BIN6 UInt64, BIN8 UInt64, PROC_CODE Int16, RESP_CODE String, PEM Int16, TRAN_CURRENCY Int16, BILL_CURRENCY Int16, TRAN_AMOUNT UInt64, TRAN_AMOUNT_RUB UInt64, TRAN_AMOUNT_STEP_100 UInt64, TRAN_AMOUNT_STEP_500 UInt64, TRAN_AMOUNT_STEP_1000 UInt64, TRAN_AMOUNT_STEP_5000 UInt64, TRAN_AMOUNT_RANGE UInt8, FEE_SIGN Int8, FEE_AMOUNT UInt64, ACQUIRING_MEMBER_ID UInt64, PROCESSING_MEMBER_ID UInt64, ISSUING_MEMBER_ID UInt64, ISSUING_MEMBER_COUNTRY UInt64, ACQUIRING_MEMBER_COUNTRY UInt64, TERMINAL_ID String, MERCHANT_ID String, TERMINAL_ID_INT Int64, MERCHANT_ID_INT Int64, MCC UInt64, TID_MID UInt64, TID_MID_MCC UInt64, ACCEPTOR_NAME_LOCATION String, SELLER_NAME String, MERCHANT_NAME String, GROUPED_MERCHANT_NAME String DEFAULT ‘-1’, MERCHANT_CITY String, MERCHANT_COUNTRY UInt64, MERCHANT_POSTAL_CODE UInt64, PARTNER_ID UInt64, TOKEN_REQUESTOR UInt64, ECI Int8, TRANSACTION_STATUS String, CARDHOLDER_PRESENCE Int16, TERMINAL_TYPE String, TRANSFER_TYPE String, BENEFIT_TYPE UInt64, CARDHOLDER_BIRTHDATE UInt64, CARDHOLDER_BIRTH_YEAR Int16, IS_G2C UInt8, IS_NON_RUB UInt8, IS_APPROVED UInt8, IS_ISSUER_RESPONSE UInt8, IS_STANDIN UInt8, IS_CALLOUT UInt8, IS_TOKEN UInt8, IS_ZERO_AMOUNT UInt8, IS_PIN_BASED UInt8, IS_AUTH_ON_US UInt8, IS_CAT UInt8, IS_COLLECTION_ONLY UInt8, IS_REPORTING_ONLY UInt8, IS_OFFLINE UInt8, RRN UInt64, AUTH_CODE String, TRN String, ACQUIRER_GATE UInt64, ISSUER_GATE UInt64, POS_DATA String, VALID String, EMV_TAGS Array(String), EMV_VALUES Array(String), ADD_DATA_TAGS Array(String), ADD_DATA_VALUES Array(String), EXT_DATA_TAGS Array(String), EXT_DATA_VALUES Array(String), PAYMENT_SOURCE_COMPANY String, PAYMENT_SOURCE_ACCOUNT String, PAYMENT_SYSTEM String DEFAULT ‘-1’, TRAFFIC_TYPE String DEFAULT ‘-1’, TRAFFIC_SOURCE String DEFAULT ‘-1’, MCC_KEY UInt64, TRAN_MC UInt8 DEFAULT 0, TRAN_VS UInt8 DEFAULT 0, TRAN_MR UInt8 DEFAULT 0, TRAN_MR_FULL UInt8 DEFAULT 0, TRAN_AMOUNT_MC UInt64 DEFAULT CAST(0, ‘UInt64’), TRAN_AMOUNT_VS UInt64 DEFAULT CAST(0, ‘UInt64’), TRAN_AMOUNT_MR UInt64 DEFAULT CAST(0, ‘UInt64’), TRAN_AMOUNT_MR_FULL UInt64 DEFAULT CAST(0, ‘UInt64’), CARD_MC UInt64 DEFAULT CAST(0, ‘UInt64’), CARD_VS UInt64 DEFAULT CAST(0, ‘UInt64’), CARD_MR UInt64 DEFAULT CAST(0, ‘UInt64’), CARD_MR_FULL UInt64 DEFAULT CAST(0, ‘UInt64’), INDEX i1 PART_KEY TYPE minmax GRANULARITY 4) ENGINE = MergeTree PARTITION BY PART_KEY ORDER BY (PS_ID, MERCHANT_ID, TERMINAL_ID) SETTINGS index_granularity = 8192

About this issue

  • Original URL
  • State: closed
  • Created 5 years ago
  • Comments: 15 (7 by maintainers)

Most upvoted comments

Reproduced on my machine.

+2
alesapin on Aug 30, 2019

@alesapin

ATTACH TABLE uasts
(
    `id` Int32, 
    `repo` String, 
    `lang` String, 
    `file` String, 
    `line` Int32, 
    `parents` Array(Int32), 
    `pkey` String, 
    `roles` Array(Int16), 
    `type` String, 
    `uptypes` Array(String), 
    `value` String
)
ENGINE = MergeTree()
ORDER BY (repo, file, id)
SETTINGS index_granularity = 8192
+1
vmarkovtsev on Aug 29, 2019

Here is the table: https://drive.google.com/file/d/1oyeD-oNuDkyKyhYXvdaXf_SeoUS4w0dK (230GB)

How to reproduce:

ALTER TABLE uasts ADD INDEX value value bloom_filter(0.01) granularity 4;
OPTIMIZE TABLE uasts FINAL;
// 3 hours later...
SELECT count(*) FROM uasts WHERE value='numpy';
+1
vmarkovtsev on Aug 27, 2019
Read more comments on GitHub
← ClickHouse: Bad size of marks file on Index
ClickHouse: Broken compatibility between 22.8.5.29 and 22.8.6.71 →