Reopenin the bug as it is seen again in 5.0-104~ocata. AnalyticsTestSanity.test_db_node_process_status_alarms failing because of the issue : After stopping the kafka service, the script expects the alarm, which is not getting generated. Expecting the alarm to be generated of type "default-global-system-config:system-defined-process-status" after executing "docker stop analyticsdatabase_kafka_1 -t 60 on nodem7 Looks like UVE is also not updated. It showed kafka as PROCESS_STATE_RUNNING. Pasting the UVE below: [root@nodem7 ~]# docker ps -a | grep kafka 1592e3c73167 10.204.217.152:5000/contrail-external-kafka:ocata-5.0-104 "/docker-entrypoin..." 7 hours ago Exited (1) 2 minutes ago analyticsdatabase_kafka_1 [root@nodem7 ~]# "NodeStatus": { "build_info": "{\"build-info\" : [{\"build-version\" : \"5.0.1\", \"build-time\" : \"2018-06-18 08:57:10.831815\", \"build-user\" : \"zuul\", \"build-hostname\" : \"centos-7-4-builder-juniper-contrail-ci-0000055808\", \"build-id\" : \"5.0-104.el7\", \"build-number\" : \"@contrail\"}]}", "installed_package_version": "5.0-104.el7", "deleted": false, "disk_usage_info": { "/dev/mapper/nodem7--vg00-lv_root": { "partition_space_available_1k": 332337864, "partition_space_used_1k": 14033452, "percentage_partition_space_used": 4, "partition_type": "ext4" } }, "__T": 1529408522111042, "running_package_version": "5.0-104.el7", "process_mem_cpu_usage": { "zookeeper": { "mem_res": 574980, "cpu_share": 0.01, "mem_virt": 577572 }, "cassandra": { "mem_res": 2721272, "cpu_share": 1.86, "mem_virt": 5909560 }, "contrail-database-nodemgr": { "mem_res": 47616, "cpu_share": 0.15, "mem_virt": 47628 }, "kafka": { "mem_res": 547848, "cpu_share": 0.0, "mem_virt": 549752 } }, "system_cpu_info": { "num_cpu": 40, "num_core_per_socket": 10, "num_thread_per_core": 2, "num_socket": 2 }, "system_mem_usage": { "used": 17315484, "cached": 13772224, "free": 230580516, "node_type": "database-node", "total": 263857016, "buffers": 2188792 }, "process_status": [ { "instance_id": "0", "module_id": "contrail-database-nodemgr", "state": "Functional", "description": null, "connection_infos": [ { "server_addrs": [ "10.204.216.103:8086" ], "status": "Up", "type": "Collector", "name": null, "description": "ClientInit to Established on EvSandeshCtrlMessageRecv" } ] } ], "all_core_file_list": [ "/var/crashes/core.contrail-query-.1.nodem7.1529387523" ], "system_cpu_usage": { "fifteen_min_avg": 2.28, "node_type": "database-node", "cpu_share": 0.18, "five_min_avg": 2.15, "one_min_avg": 2.31 }, "process_info": [ { "process_name": "kafka", "start_count": 4, "process_state": "PROCESS_STATE_RUNNING", "last_stop_time": null, "core_file_list": [ ], "last_start_time": "1529408511162401", "stop_count": 0, "last_exit_time": "1529408108967889", "exit_count": 3 }, { "process_name": "zookeeper", "start_count": 1, "process_state": "PROCESS_STATE_RUNNING", "last_stop_time": null, "core_file_list": [ ], "last_start_time": "1529363031000000", "stop_count": 0, "last_exit_time": null, "exit_count": 0 }, { "process_name": "contrail-database-nodemgr", "start_count": 1, "process_state": "PROCESS_STATE_RUNNING", "last_stop_time": null, "core_file_list": [ ], "last_start_time": "1529383306000000", "stop_count": 0, "last_exit_time": null, "exit_count": 0 }, { "process_name": "cassandra", "start_count": 1, "process_state": "PROCESS_STATE_RUNNING", "last_stop_time": null, "core_file_list": [ ], "last_start_time": "1529363031000000", "stop_count": 0, "last_exit_time": null, "exit_count": 0 } ] }, "CassandraStatusData": { "__T": 1529408511750954, "thread_pool_stats": [ [ [ { "active": 0, "pool_name": "ReadStage", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "CompactionExecutor", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "MutationStage", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "MemtableFlushWriter", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "Native-Transport-Requests", "pending": 0, "all_time_blocked": 379269 } ], "nodem7:Database:contrail-database-nodemgr:0" ], [ [ { "active": 0, "pool_name": "ReadStage", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "CompactionExecutor", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "MutationStage", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "Native-Transport-Requests", "pending": 0, "all_time_blocked": 0 }, { "active": 0, "pool_name": "MemtableFlushWriter", "pending": 0, "all_time_blocked": 0 } ], "nodem7:Config:contrail-config-nodemgr:0" ] ], "cassandra_compaction_task": { "pending_compaction_tasks": 0 } }, "ContrailConfig": { "deleted": false, "__T": 1529408511750954, "elements": { "database_node_ip_address": "\"10.204.216.96\"", "fq_name": "[\"default-global-system-config\", \"nodem7\"]", "parent_uuid": "\"a715f78f-7b3f-4acd-ac47-d1bb8d1fbe0c\"", "parent_type": "\"global-system-config\"", "perms2": "{\"owner\": \"cloud-admin\", \"owner_access\": 7, \"global_access\": 0, \"share\": []}", "id_perms": "{\"enable\": true, \"uuid\": {\"uuid_mslong\": 2094117328666119418, \"uuid_lslong\": 11968844716202857479}, \"created\": \"2018-06-19T04:33:51.815690\", \"description\": null, \"creator\": null, \"user_visible\": true, \"last_modified\": \"2018-06-19T04:33:51.815690\", \"permissions\": {\"owner\": \"admin\", \"owner_access\": 7, \"other_access\": 7, \"group\": \"admin\", \"group_access\": 7}}", "display_name": "\"nodem7\"", "uuid": "\"1d0fcc9d-8522-4cfa-a619-e0db11151407\"" } }, "UVEAlarms": { "alarms": [ { "severity": 0, "alarm_rules": { "or_list": [ { "and_list": [ { "condition": { "operation": "!=", "operand1": "NodeStatus.all_core_file_list", "variables": [ ], "operand2": { "json_value": "null" } }, "match": [ { "json_operand1_value": "[\"/var/crashes/core.contrail-query-.1.nodem7.1529387523\"]", "json_variables": { } } ] }, { "condition": { "operation": "size!=", "operand1": "NodeStatus.all_core_file_list", "variables": [ ], "operand2": { "json_value": "0" } }, "match": [ { "json_operand1_value": "[\"/var/crashes/core.contrail-query-.1.nodem7.1529387523\"]", "json_variables": { } } ] } ] } ] }, "timestamp": 1529408511510250, "ack": false, "token": "eyJ0aW1lc3RhbXAiOiAxNTI5NDA4NTExNTEwMjUwLCAiaHR0cF9wb3J0IjogNTk5NSwgImhvc3RfaXAiOiAiMTAuMjA0LjIxNi45NSJ9", "type": "default-global-system-config:system-defined-core-files", "description": "A core file has been generated on the node." } ], "__T": 1529408512654665 }, "DatabaseUsageInfo": { "__T": 1529408519048656, "database_usage": [ [ [ { "disk_space_available_1k": 332338572, "analytics_db_size_1k": 2464820, "disk_space_used_1k": 14032744 } ], "nodem7:Database:contrail-database-nodemgr:0" ], [ [ { "disk_space_available_1k": 332337780, "config_db_size_1k": 1480, "disk_space_used_1k": 14033536 } ], "nodem7:Config:contrail-config-nodemgr:0" ] ] } } Contrail alarm gen logs: 06/19/2018 05:12:14 PM [kafka.conn]: : socket disconnected 06/19/2018 05:12:14 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:14 PM [kafka.consumer.fetcher]: Fetch to node 3 failed: ConnectionError: socket disconnected 06/19/2018 05:12:14 PM [kafka.conn]: : socket disconnected 06/19/2018 05:12:14 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:14 PM [kafka.consumer.fetcher]: Fetch to node 3 failed: ConnectionError: socket disconnected 06/19/2018 05:12:14 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:14 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:14 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:14 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:15 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:15 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:15 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:15 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:15 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:15 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:15 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:15 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:16 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:16 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:16 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:16 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:16 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:16 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:16 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:16 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:17 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:17 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:17 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:17 PM [kafka.client]: Node 3 connection failed -- refreshing metadata 06/19/2018 05:12:17 PM [kafka.conn]: Connect attempt to returned error 111. Disconnecting. 06/19/2018 05:12:17 PM [kafka.client]: Node 3 connection failed -- refreshing metadata Setup details: Build : 5.0-104 DISTRO : CentOS Linux release 7.5.1804 Core SKU : ocata Config Nodes : ['nodem7', 'nodem6', 'nodem14'] Control Nodes : ['nodem7', 'nodem6', 'nodem14'] Compute Nodes : ['nodem10', 'nodem9', 'nodem8'] Openstack Node : ['nodem7', 'nodem6', 'nodem14'] WebUI Node : ['nodem7', 'nodem6', 'nodem14'] Analytics Nodes : ['nodem7', 'nodem6', 'nodem14'] Database Nodes : ['nodem7', 'nodem6', 'nodem14'] Physical Devices : ['blr-! mx1', u"'blr-mx1', '10.10.10.101'"] L