rqlite: Random database disk image is malformed errors

Can you help with this case where randomly some queries are ruturning with the {"error": "database disk image is malformed"} error?

What version are you running? v7.6.1

Are you using Docker or Kubernetes to run your system? No

Are you running a single node or a cluster? Cluster with 3 nodes.

What did you do? A python application using sqlalchemy-rqlite and pyrqlite performing simple SELECT query in just one table.

What did you expect to happen? Get the result back from the query.

What happened instead? Randomly get the following error: {“error”: “database disk image is malformed”}

Please include the Status, Nodes, and Expvar output from each node (or at least the Leader!)

Status:

curl 192.168.100.116:4001/status?pretty
{
    "build": {
        "branch": "master",
        "build_time": "2022-08-07T09:23:22-0400",
        "commit": "58568a4577940bf3adf8794c9e6e977a36e9e5db",
        "compiler": "gc",
        "version": "v7.6.1"
    },
    "cluster": {
        "addr": "192.168.100.116:4002",
        "api_addr": "192.168.100.116:4001",
        "https": "false"
    },
    "http": {
        "auth": "disabled",
        "bind_addr": "192.168.100.116:4001",
        "cluster": {
            "conn_pool_stats": {
                "192.168.100.114:4002": {
                    "idle": 4,
                    "max_open_connections": 64,
                    "open_connections": 4
                },
                "192.168.100.189:4002": {
                    "idle": 3,
                    "max_open_connections": 64,
                    "open_connections": 4
                }
            },
            "local_node_addr": "192.168.100.116:4002",
            "timeout": 30000000000
        },
        "queue": {
            "_default": {
                "batch_size": 16,
                "max_size": 128,
                "sequence_number": 0,
                "timeout": 50000000
            }
        }
    },
    "last_backup_time": "2022-11-01T03:00:04.12518951Z",
    "node": {
        "current_time": "2022-11-01T16:41:43.449966112Z",
        "start_time": "2022-10-28T17:26:47.632592627Z",
        "uptime": "95h14m55.817374557s"
    },
    "os": {
        "executable": "/usr/bin/rqlited",
        "hostname": "nodew-2",
        "page_size": 4096,
        "pid": 14912,
        "ppid": 14911
    },
    "runtime": {
        "GOARCH": "amd64",
        "GOMAXPROCS": 32,
        "GOOS": "linux",
        "num_cpu": 32,
        "num_goroutine": 62,
        "version": "go1.18"
    },
    "store": {
        "addr": "192.168.100.116:4002",
        "apply_timeout": "10s",
        "db_applied_index": 332019,
        "db_conf": {
            "memory": true,
            "fk_constraints": true
        },
        "dir": "/var/rqlited",
        "dir_size": 84744671,
        "election_timeout": "1s",
        "fsm_index": 332019,
        "heartbeat_timeout": "1s",
        "leader": {
            "addr": "192.168.100.116:4002",
            "node_id": "nodew-2"
        },
        "no_freelist_sync": false,
        "node_id": "nodew-2",
        "nodes": [
            {
                "id": "nodeh-2",
                "addr": "192.168.100.189:4002",
                "suffrage": "Voter"
            },
            {
                "id": "nodep-2",
                "addr": "192.168.100.114:4002",
                "suffrage": "Voter"
            },
            {
                "id": "nodew-2",
                "addr": "192.168.100.116:4002",
                "suffrage": "Voter"
            }
        ],
        "observer": {
            "dropped": 0,
            "observed": 3
        },
        "raft": {
            "applied_index": 332019,
            "bolt": {
                "FreePageN": 3694,
                "PendingPageN": 12,
                "FreeAlloc": 15179776,
                "FreelistInuse": 29664,
                "TxN": 91963,
                "OpenTxN": 0,
                "TxStats": {
                    "PageCount": 429033,
                    "PageAlloc": 1757319168,
                    "CursorCount": 280132,
                    "NodeCount": 136762,
                    "NodeDeref": 0,
                    "Rebalance": 16908,
                    "RebalanceTime": 14209681,
                    "Split": 8192,
                    "Spill": 136448,
                    "SpillTime": 2367516609,
                    "Write": 200584,
                    "WriteTime": 13715151494
                }
            },
            "commit_index": 332019,
            "fsm_pending": 0,
            "last_contact": 0,
            "last_log_index": 332019,
            "last_log_term": 21,
            "last_snapshot_index": 325181,
            "last_snapshot_term": 21,
            "latest_configuration": "[{Suffrage:Voter ID:nodep-2 Address:192.168.100.114:4002} {Suffrage:Voter ID:nodew-2 Address:192.168.100.116:4002} {Suffrage:Voter ID:nodeh-2 Address:192.168.100.189:4002}]",
            "latest_configuration_index": 0,
            "log_size": 41955328,
            "num_peers": 2,
            "protocol_version": 3,
            "protocol_version_max": 3,
            "protocol_version_min": 0,
            "snapshot_version_max": 1,
            "snapshot_version_min": 0,
            "state": "Leader",
            "term": 21
        },
        "request_marshaler": {
            "compression_batch": 5,
            "compression_size": 150,
            "force_compression": false
        },
        "snapshot_interval": 30000000000,
        "snapshot_threshold": 8192,
        "sqlite3": {
            "compile_options": [
                "ATOMIC_INTRINSICS=1",
                "COMPILER=gcc-9.4.0",
                "DEFAULT_AUTOVACUUM",
                "DEFAULT_CACHE_SIZE=-2000",
                "DEFAULT_FILE_FORMAT=4",
                "DEFAULT_JOURNAL_SIZE_LIMIT=-1",
                "DEFAULT_MMAP_SIZE=0",
                "DEFAULT_PAGE_SIZE=4096",
                "DEFAULT_PCACHE_INITSZ=20",
                "DEFAULT_RECURSIVE_TRIGGERS",
                "DEFAULT_SECTOR_SIZE=4096",
                "DEFAULT_SYNCHRONOUS=2",
                "DEFAULT_WAL_AUTOCHECKPOINT=1000",
                "DEFAULT_WAL_SYNCHRONOUS=1",
                "DEFAULT_WORKER_THREADS=0",
                "ENABLE_DBSTAT_VTAB",
                "ENABLE_FTS3",
                "ENABLE_FTS3_PARENTHESIS",
                "ENABLE_RTREE",
                "ENABLE_UPDATE_DELETE_LIMIT",
                "MALLOC_SOFT_LIMIT=1024",
                "MAX_ATTACHED=10",
                "MAX_COLUMN=2000",
                "MAX_COMPOUND_SELECT=500",
                "MAX_DEFAULT_PAGE_SIZE=8192",
                "MAX_EXPR_DEPTH=1000",
                "MAX_FUNCTION_ARG=127",
                "MAX_LENGTH=1000000000",
                "MAX_LIKE_PATTERN_LENGTH=50000",
                "MAX_MMAP_SIZE=0x7fff0000",
                "MAX_PAGE_COUNT=1073741823",
                "MAX_PAGE_SIZE=65536",
                "MAX_SQL_LENGTH=1000000000",
                "MAX_TRIGGER_DEPTH=1000",
                "MAX_VARIABLE_NUMBER=32766",
                "MAX_VDBE_OP=250000000",
                "MAX_WORKER_THREADS=8",
                "MUTEX_PTHREADS",
                "OMIT_DEPRECATED",
                "OMIT_LOAD_EXTENSION",
                "OMIT_SHARED_CACHE",
                "SYSTEM_MALLOC",
                "TEMP_STORE=1",
                "THREADSAFE=1"
            ],
            "conn_pool_stats": {
                "ro": {
                    "max_open_connections": 0,
                    "open_connections": 2,
                    "in_use": 0,
                    "idle": 2,
                    "wait_count": 0,
                    "wait_duration": 0,
                    "max_idle_closed": 566,
                    "max_idle_time_closed": 0,
                    "max_lifetime_closed": 0
                },
                "rw": {
                    "max_open_connections": 1,
                    "open_connections": 1,
                    "in_use": 0,
                    "idle": 1,
                    "wait_count": 0,
                    "wait_duration": 0,
                    "max_idle_closed": 0,
                    "max_idle_time_closed": 0,
                    "max_lifetime_closed": 0
                }
            },
            "db_size": 129212416,
            "mem_stats": {
                "cache_size": -2000,
                "freelist_count": 0,
                "hard_heap_limit": 0,
                "max_page_count": 1073741823,
                "page_count": 31546,
                "page_size": 4096,
                "soft_heap_limit": 0
            },
            "path": ":memory:",
            "ro_dsn": "file:/FiGbErArFfLlJdfkfsfB?mode=ro\u0026vfs=memdb\u0026_txlock=deferred\u0026_fk=true",
            "rw_dsn": "file:/FiGbErArFfLlJdfkfsfB?mode=rw\u0026vfs=memdb\u0026_txlock=immediate\u0026_fk=true",
            "version": "3.38.5"
        },
        "startup_on_disk": false,
        "trailing_logs": 10240
    }
}

Nodes:

curl 192.168.100.116:4001/nodes?pretty
{
    "nodeh-2": {
        "api_addr": "http://192.168.100.189:4001",
        "addr": "192.168.100.189:4002",
        "reachable": true,
        "leader": false,
        "time": 0.001642888
    },
    "nodep-2": {
        "api_addr": "http://192.168.100.114:4001",
        "addr": "192.168.100.114:4002",
        "reachable": true,
        "leader": false,
        "time": 0.001582912
    },
    "nodew-2": {
        "api_addr": "http://192.168.100.116:4001",
        "addr": "192.168.100.116:4002",
        "reachable": true,
        "leader": true,
        "time": 0.00006786
    }
}

Expvar:

curl 192.168.100.116:4001/debug/vars
{
"cluster": {"num_execute_req": 21509, "num_get_node_api_req": 2296, "num_get_node_api_req_local": 1182, "num_get_node_api_resp": 2296, "num_query_req": 112289},
"cmdline": ["/usr/bin/rqlited","-fk=true","-node-id","nodew-2","-http-addr","192.168.100.116:4001","-raft-addr","192.168.100.116:4002","-bootstrap-expect","3","-join","http://192.168.100.116:4001,http://192.168.100.114:4001,http://192.168.100.189:4001","/var/rqlited"],
"db": {"execute_transactions": 40001, "execution_errors": 1, "executions": 40002, "queries": 164992, "query_errors": 15, "query_transactions": 0},
"http": {"authFail": 0, "authOK": 65019, "backups": 12, "executions": 10294, "joins": 0, "leader_not_found": 0, "loads": 0, "notifies": 0, "num_readyz": 1144, "num_status": 2, "queries": 53528, "queued_executions": 0, "queued_executions_failed": 0, "queued_executions_ok": 0, "queued_executions_wait": 0, "remote_executions": 161, "remote_queries": 843},
"memstats": {"Alloc":5877768,"TotalAlloc":58410672832,"Sys":1250371352,"Lookups":0,"Mallocs":448749443,"Frees":448721958,"HeapAlloc":5877768,"HeapSys":1166606336,"HeapIdle":1157619712,"HeapInuse":8986624,"HeapReleased":1154220032,"HeapObjects":27485,"StackInuse":3604480,"StackSys":3604480,"MSpanInuse":491232,"MSpanSys":17233920,"MCacheInuse":38400,"MCacheSys":46800,"BuckHashSys":1729007,"GCSys":53580296,"OtherSys":7570513,"NextGC":10118288,"LastGC":1667321352097784485,"PauseTotalNs":1900149399,"PauseNs":[115601,141338,102226,140742,133325,102017,85790,139333,88605,91160,124217,102892,131807,174539,118341,128988,134937,108283,144730,105161,107145,110191,121275,110677,143066,123505,104821,133136,111502,103979,131773,100611,91631,132327,122775,151810,171199,88860,150985,142711,119758,121693,152044,149378,96463,93229,85040,114028,93450,155367,142086,109016,184573,153716,138629,117482,114063,143868,87134,103374,117381,146351,128852,116867,111673,131205,114590,188150,201372,120573,97841,126588,115544,139381,117155,115755,129005,97070,156077,153443,162651,96896,84659,114337,105722,97995,109939,92664,89050,127997,90148,101404,108345,116850,97484,83734,110461,82684,113191,122000,84732,92627,80491,81548,100423,118579,120854,109330,108833,111159,114634,118509,111001,99303,108104,115408,104605,90924,91487,94839,117224,154048,100023,101576,106167,116064,136277,144991,96687,94546,102006,116674,120768,112272,140898,106286,105060,105517,95563,123494,99914,117605,99831,100371,104078,118491,98807,102337,120118,143598,116640,108378,123267,150354,160713,131751,118198,94555,126112,133958,139208,95251,104930,103016,166341,254268,138391,110508,104297,168643,113313,125604,85942,200608,157345,77744,130682,96529,103470,121816,126064,127075,120373,108314,105796,83776,135351,102572,116496,149920,134474,112555,97864,124826,111746,104015,79432,173966,122732,103729,115700,105048,91718,101058,109053,104194,118503,193869,84386,150663,100871,150374,115895,105968,125450,159866,111838,90968,133471,98444,96435,108621,107147,128511,124097,125273,114787,109758,82270,103898,111645,103270,103368,119237,115831,123950,139330,223225,144705,112503,137238,112699,118674,184792,122124,88639,118498,127474,136092,157636,113028,109714,112668,109917,103970,162788],"PauseEnd":[1667317808988571856,1667317809426336191,1667317809758711745,1667317810445360146,1667317812681230792,1667317814680659415,1667317844885309764,1667317845429294851,1667317849118110037,1667317850667376784,1667317851383893248,1667317852219180243,1667317852777320057,1667317853571394705,1667317854293679191,1667317854971331844,1667317855188774780,1667317855624011011,1667317855990764755,1667317856410095597,1667317856713575800,1667317857204436438,1667317857462645680,1667317857938967444,1667317858217350441,1667317858512428955,1667317858794935106,1667317859211946444,1667317859756777546,1667317860182908540,1667317860597929442,1667317860707317466,1667317861605597717,1667317861949770832,1667317862470779391,1667317862836589350,1667317863231728963,1667317863709813258,1667317864110715404,1667317865208453613,1667317865911386944,1667317866705960496,1667317868429652915,1667317878233248165,1667317889637092181,1667317904390389728,1667317937340229624,1667317947433777656,1667317947979898250,1667317949010602794,1667317950765204741,1667317951782251602,1667317956312864301,1667317959380690178,1667317960939840531,1667318019496513374,1667318039074033692,1667318054293873234,1667318081483307212,1667318135001679783,1667318164004340485,1667318185322430786,1667318197869209823,1667318240173884699,1667318254591167300,1667318304136369419,1667318305325884928,1667318306539150218,1667318315210917463,1667318323424950228,1667318387533599721,1667318410265211332,1667318461540891817,1667318533126814634,1667318602535487074,1667318604244268617,1667318635221884110,1667318682013656646,1667318703091079732,1667318731442098020,1667318742408470168,1667318777506018969,1667318783972500668,1667318785463378757,1667318812113509614,1667318858575258081,1667318868843406873,1667318923098432679,1667318996573109861,1667319071113972941,1667319094661912469,1667319098943420906,1667319125987446792,1667319181068877022,1667319204082944420,1667319218245222390,1667319224408610269,1667319301225433692,1667319326926768377,1667319369598260321,1667319370864557315,1667319373691249763,1667319376535729162,1667319401015897835,1667319430610054357,1667319483620377698,1667319494119639406,1667319508868175340,1667319562860910393,1667319595639522786,1667319642844869636,1667319648566924728,1667319695779416106,1667319728540907059,1667319774253250378,1667319828606216540,1667319879464652957,1667319934243911904,1667319935913681365,1667319937465896888,1667319961512223725,1667319985790204531,1667320014534884502,1667320036951741050,1667320044804705289,1667320084256318709,1667320086595294914,1667320087855855866,1667320092153829202,1667320101857041045,1667320193148194981,1667320263481793353,1667320275277315133,1667320318624856362,1667320373236986464,1667320406929593145,1667320462500285205,1667320533766583794,1667320586203190390,1667320628629314397,1667320682846664629,1667320731960635046,1667320761990454527,1667320787302274236,1667320841764508344,1667320907987956139,1667320963248174745,1667320986690011309,1667321015421821017,1667321061469444355,1667321085388450526,1667321119695180334,1667321120088074700,1667321148784661376,1667321161365191891,1667321163303309025,1667321164220002458,1667321165712150981,1667321167105381907,1667321184810430429,1667321240466393922,1667321301625886733,1667321321523945188,1667321323973429356,1667321324874023198,1667321326837298468,1667321329163477023,1667321330863963509,1667321331848687810,1667321332391085056,1667321333038880858,1667321333555938238,1667321334381955804,1667321335151015480,1667321336320111936,1667321337459935940,1667321337660125738,1667321338199944321,1667321338957390274,1667321339456422195,1667321340059348592,1667321340604417210,1667321341356999421,1667321342281568120,1667321343789259603,1667321344199338345,1667321345353314579,1667321346877030368,1667321349997855552,1667321352097784485,1667317235336438600,1667317235594678471,1667317235967244233,1667317236426428151,1667317237101314974,1667317237319629326,1667317237954993712,1667317239338297796,1667317239854242763,1667317240181751378,1667317240811843824,1667317241231857232,1667317241907962119,1667317242712194940,1667317243172775081,1667317243876055029,1667317244105669938,1667317244903338755,1667317245352393038,1667317245802453985,1667317246398232122,1667317247154864192,1667317248055687950,1667317248292485335,1667317249247475206,1667317249703555392,1667317253003310954,1667317255199965622,1667317255926164468,1667317258197622597,1667317265227674859,1667317301152119376,1667317320571384312,1667317321894496200,1667317323190756603,1667317324123407186,1667317324987827525,1667317326457079231,1667317327577447314,1667317329338715039,1667317332516847296,1667317343817699968,1667317354274988689,1667317383552923648,1667317391063700818,1667317406914278558,1667317435289882604,1667317452631417812,1667317458719885743,1667317495540460373,1667317536717864035,1667317557241992422,1667317623066155369,1667317686660504895,1667317722257456445,1667317725954616738,1667317766501861208,1667317782447269530,1667317801541498601,1667317804217728114,1667317804819891159,1667317805254213951,1667317805973778163,1667317806831678984,1667317807951751003,1667317808219174758],"NumGC":14782,"NumForcedGC":0,"GCCPUFraction":0.000017445228744738093,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":1726395,"Frees":1725012},{"Size":16,"Mallocs":100906050,"Frees":100902027},{"Size":24,"Mallocs":74656261,"Frees":74652591},{"Size":32,"Mallocs":31421455,"Frees":31419907},{"Size":48,"Mallocs":27136355,"Frees":27134001},{"Size":64,"Mallocs":26235030,"Frees":26233373},{"Size":80,"Mallocs":85331291,"Frees":85327571},{"Size":96,"Mallocs":33588578,"Frees":33584142},{"Size":112,"Mallocs":362368,"Frees":362219},{"Size":128,"Mallocs":745592,"Frees":745456},{"Size":144,"Mallocs":9613885,"Frees":9613260},{"Size":160,"Mallocs":185096,"Frees":184997},{"Size":176,"Mallocs":204956,"Frees":204317},{"Size":192,"Mallocs":127771,"Frees":127734},{"Size":208,"Mallocs":963978,"Frees":963856},{"Size":224,"Mallocs":320715,"Frees":320705},{"Size":240,"Mallocs":95852,"Frees":95846},{"Size":256,"Mallocs":143368,"Frees":143311},{"Size":288,"Mallocs":154842,"Frees":154504},{"Size":320,"Mallocs":140598,"Frees":140532},{"Size":352,"Mallocs":600177,"Frees":600039},{"Size":384,"Mallocs":71452,"Frees":71429},{"Size":416,"Mallocs":1926159,"Frees":1924953},{"Size":448,"Mallocs":24252,"Frees":24219},{"Size":480,"Mallocs":43847,"Frees":43839},{"Size":512,"Mallocs":9207775,"Frees":9207207},{"Size":576,"Mallocs":49963,"Frees":49842},{"Size":640,"Mallocs":17227,"Frees":17207},{"Size":704,"Mallocs":42702,"Frees":42687},{"Size":768,"Mallocs":41451,"Frees":41447},{"Size":896,"Mallocs":68435,"Frees":68414},{"Size":1024,"Mallocs":68978,"Frees":68920},{"Size":1152,"Mallocs":67525,"Frees":67512},{"Size":1280,"Mallocs":67801,"Frees":67788},{"Size":1408,"Mallocs":93856,"Frees":93852},{"Size":1536,"Mallocs":37231,"Frees":37230},{"Size":1792,"Mallocs":9659,"Frees":9648},{"Size":2048,"Mallocs":19722,"Frees":19716},{"Size":2304,"Mallocs":104562,"Frees":104549},{"Size":2688,"Mallocs":52935,"Frees":52925},{"Size":3072,"Mallocs":9302,"Frees":9300},{"Size":3200,"Mallocs":3177,"Frees":3176},{"Size":3456,"Mallocs":4301,"Frees":4299},{"Size":4096,"Mallocs":139433,"Frees":139391},{"Size":4864,"Mallocs":40237,"Frees":40235},{"Size":5376,"Mallocs":360,"Frees":356},{"Size":6144,"Mallocs":4485,"Frees":4481},{"Size":6528,"Mallocs":2326,"Frees":2326},{"Size":6784,"Mallocs":1603,"Frees":1603},{"Size":6912,"Mallocs":795,"Frees":795},{"Size":8192,"Mallocs":7717,"Frees":7710},{"Size":9472,"Mallocs":7828,"Frees":7793},{"Size":9728,"Mallocs":1550,"Frees":1550},{"Size":10240,"Mallocs":3112,"Frees":3110},{"Size":10880,"Mallocs":3438,"Frees":3438},{"Size":12288,"Mallocs":1062,"Frees":1062},{"Size":13568,"Mallocs":973,"Frees":973},{"Size":14336,"Mallocs":598,"Frees":598},{"Size":16384,"Mallocs":1565,"Frees":1560},{"Size":18432,"Mallocs":1563,"Frees":1563}]},
"mux": {"num_connections_handled": 10, "num_unregistered_handlers": 0},
"proto": {"num_compressed_bytes": 11466091, "num_compressed_requests": 31641, "num_compression_misses": 0, "num_precompressed_bytes": 16462122, "num_requests": 0, "num_uncompressed_bytes": 8, "num_uncompressed_requests": 1},
"store": {"leader_changes_dropped": 0, "leader_changes_observed": 0, "num_backups": 12, "num_compressed_commands": 31641, "num_ignored_joins": 0, "num_joins": 0, "num_recoveries": 0, "num_removed_before_joins": 0, "num_restores": 1, "num_snapshots": 4, "num_uncompressed_commands": 1, "snapshot_create_duration": 291, "snapshot_db_ondisk_size": 21502211, "snapshot_db_serialized_size": 128036864, "snapshot_persist_duration": 6115}
}