memcached: Error: Server at 127.0.0.1:11211 not available

LOG

Error: Server at 127.0.0.1:11211 not available
at Client.memcachedCommand [as command] (/home/hello.com/node_modules/memcached/lib/memcached.js:306:70)
at Client.get (/home/hello.com/node_modules/memcached/lib/memcached.js:830:10)

But Memcached active.

# service memcached status
● memcached.service - memcached daemon
   Loaded: loaded (/lib/systemd/system/memcached.service; disabled)
   Active: active (running) since Thu 2017-03-23 08:35:35 EDT; 10s ago
 Main PID: 1538 (memcached)
   CGroup: /system.slice/memcached.service
           └─1538 /usr/bin/memcached -m 64 -p 11211 -u memcache -l 127.0.0.1

Mar 23 08:35:35 trampamp systemd[1]: Starting memcached daemon...
Mar 23 08:35:35 trampamp systemd[1]: Started memcached daemon.
# telnet 127.0.0.1 11211
Trying 127.0.0.1...
Connected to 127.0.0.1.
Escape character is '^]'.
stats
STAT pid 1538
STAT uptime 876
STAT time 1490273409
STAT version 1.4.21
STAT libevent 2.0.21-stable
STAT pointer_size 64
STAT rusage_user 0.025996
STAT rusage_system 0.006998
STAT curr_connections 5
STAT total_connections 8
STAT connection_structures 6
STAT reserved_fds 20
STAT cmd_get 0
STAT cmd_set 0
STAT cmd_flush 2
STAT cmd_touch 0
STAT get_hits 0
STAT get_misses 0
STAT delete_misses 0
STAT delete_hits 0
STAT incr_misses 0
STAT incr_hits 0
STAT decr_misses 0
STAT decr_hits 0
STAT cas_misses 0
STAT cas_hits 0
STAT cas_badval 0
STAT touch_hits 0
STAT touch_misses 0
STAT auth_cmds 0
STAT auth_errors 0
STAT bytes_read 46
STAT bytes_written 15
STAT limit_maxbytes 67108864
STAT accepting_conns 1
STAT listen_disabled_num 0
STAT threads 4
STAT conn_yields 0
STAT hash_power_level 16
STAT hash_bytes 524288
STAT hash_is_expanding 0
STAT malloc_fails 0
STAT bytes 0
STAT curr_items 0
STAT total_items 0
STAT expired_unfetched 0
STAT evicted_unfetched 0
STAT evictions 0
STAT reclaimed 0
STAT crawler_reclaimed 0
STAT lrutail_reflocked 0
END

About this issue

  • Original URL
  • State: open
  • Created 7 years ago
  • Reactions: 8
  • Comments: 16

Most upvoted comments

Looks like I got it!
this is how my memcached is started

memcached -p 11212 -m 256 -c 1024 -I 20m

and this was the client creation

new Memcached(
    options.host + ':' + options.port, {
        maxValue: 33554432
    }
);

when i corrected maxValue to the value <= 20Mb, for example 10485760
in logs instead of those unexpected errors i sometimes see for too big items

Error: The length of the value is greater than 10485760

which is perfectly fine and all other write operations succeed

I think I managed to consistently reproduce Error: Connection readyState is set to readOnly locally. Basically, you need memcached server running + stress test it with parallel writes. For me it starts to break at 40 parallel writes (but below I have it set to 100)

Considering that nothing else uses this memcached server & I have only one connection instance, I think its a library fault that it doesn’t try to queue writes… nor limit how many writes can do at a time & instead seems to just kill memcache.

Steps (using jest / integration test):

const MIN_FAILING_THREADS = 100;
it(`native memcache fails at ${MIN_FAILING_THREADS} concurrent writes`, async () => {
	const Memcached = require('memcached');

	// ACT
	const ops = [];

	const connection = new Memcached(['localhost:11211'], {
		// timeout: 500, the time after which Memcached sends a connection timeout (in milliseconds)
		timeout: 500,
		// retries: 5, the number of socket allocation retries per request.
		retries: 2,
		// failures: 5, the number of failed-attempts to a server before it is regarded as 'dead'.
		failures: 5,
		// retry: 30000, the time between a server failure and an attempt to set it up back in service.
		// the maximum size of the connection pool.
		poolSize: 200,
		// if true, authorizes the automatic removal of dead servers from the pool.
		remove: true,
		idle: 5000,
		maxValue: 524288, // default is 1048576
		encoding: 'binary',
	});

	connection.on('failure', async () => {
		console.error('Lost connection');
	});

	for (let i = 0; i < MIN_FAILING_THREADS; i++) {
		ops.push(
			new Promise((resolve, reject) => {
				connection.set(`x${i}`, i, 2, (err, result) => {
					if (err) {
						return reject(err);
					}

					return resolve(result);
				});
			}),
		);
	}

	// overload memcache
	Promise.all(ops);

	const result = await new Promise((resolve, reject) =>
		connection.getMulti(['x1'], (err, data) => {
			if (err) {
				reject(err);
			}

			resolve(data);
		}),
	);

	// ASSERT
        expect(result).toEqual({ x1: 1 });
});