root@mgt:/home/jeffreyr# journalctl -fu snap.maas.pebble -t maas-temporal
May 29 10:56:31 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T10:56:31.726Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:f2b8310e-6e8f-47f5-a81b-1d4c48e9ea9d","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 10:56:31 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T10:56:31.726Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:0c6fe162-ecb7-4473-a0ec-2189f0914c3e","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 11:15:31 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T11:15:31.746Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:1982de8a-96fb-455a-81b5-ed0af235f6a8","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 11:53:04 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T11:53:04.223Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:1d1e3068-776a-4b07-b393-b5c447ffa35a","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 11:53:04 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T11:53:04.223Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:6dbbcaf8-7608-47a4-9d3c-5dba80c82e7a","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 11:53:04 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T11:53:04.223Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:fc6df88b-c8ed-4873-9f44-4ce253402722","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 11:55:03 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T11:55:03.219Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:d7a3fb57-c8aa-4a98-9ef4-5c099c65a9a6","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 12:50:31 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T12:50:31.724Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:9fad7bcb-1d02-4753-9343-1c7f75e62725","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}
May 29 14:13:31 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T14:13:31.850Z","msg":"unable to add metrics baggage to gRPC trailer","error":"rpc error: code = Internal desc = transport: SendHeader called multiple times","logging-call-at":"grpc.go:143","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/common/metrics.NewServerMetricsTrailerPropagatorInterceptor.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/metrics/grpc.go:143\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:1195\ngo.temporal.io/server/common/metrics.NewServerMetricsContextInjectorInterceptor.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/metrics/grpc.go:66\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:1195\ngo.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:344\ngoogle.golang.org/grpc.getChainUnaryHandler.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:1195\ngo.temporal.io/server/common/rpc.ServiceErrorInterceptor\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/rpc/grpc.go:145\ngoogle.golang.org/grpc.chainUnaryInterceptors.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:1186\ngo.temporal.io/server/api/matchingservice/v1._MatchingService_PollActivityTaskQueue_Handler\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/api/matchingservice/v1/service.pb.go:480\ngoogle.golang.org/grpc.(*Server).processUnaryRPC\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:1376\ngoogle.golang.org/grpc.(*Server).handleStream\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:1753\ngoogle.golang.org/grpc.(*Server).serveStreams.func1.1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/vendor/google.golang.org/grpc/server.go:998"}
May 29 15:52:31 mgt maas-temporal[2739]: {"level":"error","ts":"2024-05-29T15:52:31.740Z","msg":"Persistent store operation failure","service":"matching","component":"matching-engine","wf-task-queue-name":"mgt:a205b060-1963-4350-9ae9-610756e1163a","wf-task-queue-type":"Workflow","wf-namespace":"default","store-operation":"update-task-queue","error":"context canceled","logging-call-at":"task_reader.go:214","stacktrace":"go.temporal.io/server/common/log.(*zapLogger).Error\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/common/log/zap_logger.go:156\ngo.temporal.io/server/service/matching.(*taskReader).getTasksPump\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/service/matching/task_reader.go:214\ngo.temporal.io/server/internal/goro.(*Group).Go.func1\n\t/build/temporal-8dYnj9/temporal-1.22.5/src/internal/goro/group.go:58"}