The following error keeps popping up even though LCM and nodetool status show all nodes in cluster as up and healthy.
ERROR [dsefs-netty-worker-4] 2020-03-19 10:55:49,307 AbstractDigestRestServerAuthProvider.scala:103 - Could not authenticate request DefaultHttpRequest(decodeResult: success, version: HTTP/1.1)
GET /info HTTP/1.1
authorization: InCluster charset=utf-8,username="AAljYXNzYW5kcmFBSowwKPMR6r05MYczFAoe",realm="default",nonce="qC1D0LXz2ylp/NnYWgFcM/RJr3X5vaRmhGZ9zCth",nc=00000001,cnonce="40kZmRZuuf955Ag1UOiOvWTzprplQl+p1hPPeub7",digest-uri="http/default",maxbuf=65536,response=c3253e246ab829db644a2e81ae9a7220,qop=auth,authzid="AAljYXNzYW5kcmFBSowwKPMR6r05MYczFAoe"
javax.security.sasl.SaslException: DIGEST-MD5: cannot acquire password for AAljYXNzYW5kcmFBSowwKPMR6r05MYczFAoe in realm : default
at com.sun.security.sasl.digest.DigestMD5Server.validateClientResponse(DigestMD5Server.java:603)
at com.sun.security.sasl.digest.DigestMD5Server.evaluateResponse(DigestMD5Server.java:244)
at com.datastax.bdp.dsefs.auth.AbstractDigestRestServerAuthProvider.authenticateRequest(AbstractDigestRestServerAuthProvider.scala:76)
at com.datastax.bdp.dsefs.auth.AbstractDigestRestServerAuthProvider.authenticate(AbstractDigestRestServerAuthProvider.scala:99)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$stateMachine$macro$434$1.apply(RestServerAuthHandler.scala:42)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$stateMachine$macro$434$1.apply$mcV$sp(RestServerAuthHandler.scala:40)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$stateMachine$macro$434$1.apply(RestServerAuthHandler.scala:40)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$stateMachine$macro$434$1.apply(RestServerAuthHandler.scala:40)
at scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
at scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
at com.datastax.bdp.fs.exec.SameThreadExecutionContext$class.executeInSameThread(SameThreadExecutionContext.scala:51)
at com.datastax.bdp.fs.exec.SameThreadExecutionContext$class.execute(SameThreadExecutionContext.scala:73)
at com.datastax.bdp.fs.exec.SerialExecutionContextProvider$$anon$5$$anon$1.execute(SerialExecutionContextProvider.scala:24)
at scala.concurrent.impl.Future$.apply(Future.scala:31)
at scala.concurrent.Future$.apply(Future.scala:494)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler.com$datastax$bdp$fs$rest$server$auth$RestServerAuthHandler$$authenticate(RestServerAuthHandler.scala:40)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$$anonfun$channelRead$1$stateMachine$macro$440$1.apply(RestServerAuthHandler.scala:105)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$$anonfun$channelRead$1$stateMachine$macro$440$1.apply$mcV$sp(RestServerAuthHandler.scala:89)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$$anonfun$channelRead$1$stateMachine$macro$440$1.apply(RestServerAuthHandler.scala:89)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$$anonfun$channelRead$1$stateMachine$macro$440$1.apply(RestServerAuthHandler.scala:89)
at scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)
at scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)
at com.datastax.bdp.fs.exec.SameThreadExecutionContext$class.executeInSameThread(SameThreadExecutionContext.scala:51)
at com.datastax.bdp.fs.exec.SameThreadExecutionContext$class.execute(SameThreadExecutionContext.scala:73)
at com.datastax.bdp.fs.exec.SerialExecutionContextProvider$$anon$5$$anon$1.execute(SerialExecutionContextProvider.scala:24)
at scala.concurrent.impl.Future$.apply(Future.scala:31)
at scala.concurrent.Future$.apply(Future.scala:494)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$$anonfun$channelRead$1.apply(RestServerAuthHandler.scala:89)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler$$anonfun$channelRead$1.apply(RestServerAuthHandler.scala:89)
at com.datastax.bdp.fs.rest.util.FutureQueue$$anonfun$enqueue$1.apply(FutureQueue.scala:32)
at com.datastax.bdp.fs.rest.util.FutureQueue$$anonfun$enqueue$1.apply(FutureQueue.scala:32)
at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32)
at com.datastax.bdp.fs.exec.SameThreadExecutionContext$class.executeInSameThread(SameThreadExecutionContext.scala:51)
at com.datastax.bdp.fs.exec.SameThreadExecutionContext$class.execute(SameThreadExecutionContext.scala:73)
at com.datastax.bdp.fs.exec.SerialExecutionContextProvider$$anon$5$$anon$1.execute(SerialExecutionContextProvider.scala:24)
at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40)
at scala.concurrent.impl.Promise$DefaultPromise.scala$concurrent$impl$Promise$DefaultPromise$$dispatchOrAddCallback(Promise.scala:280)
at scala.concurrent.impl.Promise$DefaultPromise.onComplete(Promise.scala:270)
at com.datastax.bdp.fs.rest.util.FutureQueue.enqueue(FutureQueue.scala:32)
at com.datastax.bdp.fs.rest.server.auth.RestServerAuthHandler.channelRead(RestServerAuthHandler.scala:88)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at com.datastax.bdp.fs.rest.server.StartTlsServerHandler.handleRequest(StartTlsServerHandler.scala:116)
at com.datastax.bdp.fs.rest.server.StartTlsServerHandler.channelRead0(StartTlsServerHandler.scala:166)
at com.datastax.bdp.fs.rest.server.StartTlsServerHandler.channelRead0(StartTlsServerHandler.scala:34)
at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:241)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelRead(CombinedChannelDuplexHandler.java:438)
at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:310)
at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:284)
at io.netty.channel.CombinedChannelDuplexHandler.channelRead(CombinedChannelDuplexHandler.java:253)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:340)
at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1434)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:362)
at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:348)
at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:965)
at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163)
at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:647)
at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:582)
at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:499)
at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:461)
at io.netty.util.concurrent.SingleThreadEventExecutor$5.run(SingleThreadEventExecutor.java:884)
at java.lang.Thread.run(Thread.java:748)
