nodejs-spanner: Error 4: Deadline for Transaction exceeded / Transaction outcome unknown

@crwilcox the following code could easily reproduce the deadline transaction exceeded error, please help to check (it may need to take a long time to encounter the error, just keep it running for 1 hour):

note that even the simplest transaction.run without querying any table occasionally encounter this.

That means doing any transaction.run inside the transaction context is potentially affected. (we could record an error rate ~ 0.2% - 0.01%)

'use strict';

const {Spanner} = require('@google-cloud/spanner');
const readline = require('readline');

const spanner = new Spanner({
	projectId: '<your project>'
});

const database = spanner
	.instance('<your instance>')
	.database('<your db>', {
		keepAlive: 5,
	});

let total = 0;
let errors = [];

async function main() {
	while (true) {
		try {
			await test();
		} catch (e) {
			e.time = new Date();
			console.error(e)
			errors.push(e);
		} finally {
			total += 1;
			readline.clearLine(process.stdout, 0)
			readline.cursorTo(process.stdout, 0, null);
			process.stdout.write(`Total: ${total}, Errors: ${errors.length}, Error rate: ${((errors.length / total) * 100).toFixed(6) + '%'}`)
		}
	}
}

main().catch(console.error);

async function test() {
	return new Promise((res, rej) => {
		database.runTransaction(async (err, transaction) => {
			if (err) return rej(err);
			try {
				await transaction.run({sql: 'select 1 as ok'})
				await transaction.commit();
				return res(true)
			} catch (e) {
				return rej(e);
			}
		});
	})
}

Hi there, we also encounter the same error randomly. We got message Deadline for Transaction exceeded. and Aborted due to transient fault. But according to our APM our request did not take longer than 10s, so didn’t look like it exceed the deadline.

Our current logging didn;t log it in detail, we will add better logging for this error.

@crwilcox When this error throw there is no retry info key in the metadata, i tried the same flow using golang, so far no error occurs.

https://gist.github.com/eric4545/c514128a625172cf5c6969ae74cd3f83

same script won’t trigger any errors in ruby sdk as well. (correct me if me usage was wrong)

require "google/cloud/spanner"

spanner = Google::Cloud::Spanner.new project_id: 'aftership-test'

db = spanner.client "aftership-test-1", "test_core_aftershipapi_com"

total = 0
errors = 0

while true do
  begin
    db.transaction do |tx|
      results = tx.execute "SELECT 1 as OK"

      total += results.rows.count
    end
    print "Total query: #{total},  errors: #{errors}\r"
    $stdout.flush
  rescue Interrupt => e
    print_exception(e, true)
  rescue Exception => e
    errors += 1
    print_exception(e, false)
  end
end