question

We are using node.js serverless solution for our application and AWS Keyspaces with a single-region strategy as the main database. Cassandra-driver is used as a node.js driver.

For some reason we are getting ERROR NoHostAvailableError: All host(s) tried for query failed. First host tried, {IP:}9142: DriverError: Socket was closed pretty often, but absolutely randomly.

Currently, we have near 20 handlers, each of them uses its own connection(due to the serverless nature). Connect function described below:

const getClient = () => {
  const authProvider = new PlainTextAuthProvider(IAM_USER, IAM_PW);
  const certFilePath = path.join(__dirname, 'AmazonRootCA1.pem');
 
  const sslOptions = {
    cert: fs.readFileSync(certFilePath),
    host: AUTH_HOST,
    rejectUnauthorized: true,
  };
 
  const clientOptions = {
    contactPoints: CONTACT_POINTS,
    localDataCenter: LOCAL_DC,
    keyspace: KEYSPACE,
 
    authProvider: authProvider,
    sslOptions: sslOptions,
    pooling: { heartBeatInterval: 0, warmup: false },
 
    protocolOptions: {
      port: 9142,
    },
    queryOptions: { consistency: types.consistencies.localQuorum },
  };
 
  const client = new Client(clientOptions);
 
  if (IS_ENABLE_DEBUG_LOGGING) {
    client.on('log', (level, className, message) => {
      if (level !== 'verbose') {
        console.log('Cassandra driver log event:', level, className, message);
      }
    });
  }
 
  return client;
};
 
let cachedClient = null;
 
const connect = async () => {
  if (!cachedClient) {
    console.log('DB optimization: first connect');
  }
 
  if (cachedClient) {
    console.log(
      'DB optimization: connect exists, connected hosts',
      cachedClient.getState().getConnectedHosts().length,
      cachedClient.getState().getConnectedHosts(),
      cachedClient.getState(),
    );
  }
 
  if (cachedClient && cachedClient.getState().getConnectedHosts().length === 0) {
    console.log('DB optimization: connect exists, but connected hosts array empty, need reconnect');
  }
 
  if (cachedClient && cachedClient.getState().getConnectedHosts().length > 0) {
    console.log('DB optimization: connect exists and connected hosts array more than 0');
 
    return cachedClient;
  }
 
  try {
    const client = getClient(options, withoutKeyspace);
 
    await client.connect();
 
    console.log('Connected to the DSE cluster, discovered %d nodes', client.hosts.length);
 
    cachedClient = client;
 
    return client;
  } catch (error) {
    console.error('There was an error trying to connect', error);
    throw error;
  }
};
 
module.exports = {
  connect,
};

We are caching connection for each handler because initial Keyspaces connect can take near 2-6 seconds, so it does not make sense to open(and then close) a new connection per each request.

Currently, we have only one keyspace, but we are planning to separate tables to a different keyspace by their domains, but we don't think that this is the root cause.

Any suggestions on how to fix this issue?

Thanks!