Build 2739: phyif-bandwidth alarm raised just after provisioning

[Ankit Jain]

Seeing phyif-bandwidth with empty rules just after provisioning.

Sigma seems to be ok but still alarm with empty rules is getting raised.

Please check @nodeg13:8081

vrouter: [
{
name: "nodeh3",
value: {
UVEAlarms: {
alarms: [
{
severity: 4,
ack: false,
timestamp: 1467976450508934,
rules: [ ],
token: "eyJ0aW1lc3RhbXAiOiAxNDY3OTc2NDUwNTA4OTM0LCAiaHR0cF9wb3J0IjogNTk5NSwgImhvc3RfaXAiOiAiMTAuMjA0LjIxNi4xNyJ9",
type: "phyif-bandwidth"
}
]
}
}
},
{
name: "nodea21",
value: {
UVEAlarms: {
alarms: [
{
severity: 4,
ack: false,
timestamp: 1467976359468904,
rules: [ ],
token: "eyJ0aW1lc3RhbXAiOiAxNDY3OTc2MzU5NDY4OTA0LCAiaHR0cF9wb3J0IjogNTk5NSwgImhvc3RfaXAiOiAiMTAuMjA0LjIxNy41MyJ9",
type: "phyif-bandwidth"
}
]
}
}
}
]
}

Pasting the output of http://nodeg13:8081/analytics/uves/vrouter/nodeh3?flat below:

{
ComputeCpuState: {
cpu_info: [
{
mem_res: 191764,
cpu_share: 1.66667,
used_sys_mem: 964556,
mem_virt: 893012,
one_min_cpuload: 0.195
}
]
},
NodeStatus: {
system_mem_cpu_usage: {
cpu_share: 2.5,
cpu_load: {
fifteen_min_avg: 0.37,
five_min_avg: 0.61,
one_min_avg: 0.66
},
mem_info: {
cached: 454712,
total: 65683280,
used: 1469236,
free: 64214044,
buffers: 50328
}
},
deleted: false,
disk_usage_info: [
{
partition_space_available_1k: 389399092,
partition_space_used_1k: 3806464,
percentage_partition_space_used: 1,
partition_name: "/dev/mapper/nodeh3--vg-root",
partition_type: "ext4"
},
{
partition_space_available_1k: 160850,
partition_space_used_1k: 67681,
percentage_partition_space_used: 30,
partition_name: "/dev/sda1",
partition_type: "ext2"
}
],
process_mem_cpu_usage: [
{
module_id: "contrail-vrouter-agent",
mem_res: 191764,
cpu_share: 0,
mem_virt: 893012,
inst_id: "0"
},
{
module_id: "contrail-vrouter-nodemgr",
mem_res: 30884,
cpu_share: 0,
mem_virt: 173636,
inst_id: "0"
}
],
system_cpu_info: {
num_cpu: 4,
num_core_per_socket: 4,
num_thread_per_core: 1,
num_socket: 1
},
process_status: [
{
instance_id: "0",
module_id: "contrail-vrouter-nodemgr",
state: "Functional",
description: null,
connection_infos: [
{
server_addrs: [
"10.204.216.17:8086"
],
status: "Up",
type: "Collector",
name: null,
description: "ClientInit to Established on EvSandeshCtrlMessageRecv"
},
{
server_addrs: [
"10.204.217.53:5998"
],
status: "Up",
type: "Discovery",
name: "Collector",
description: "Subscribe Response"
}
]
},
{
instance_id: "0",
module_id: "contrail-vrouter-agent",
state: "Functional",
description: null,
connection_infos: [
{
server_addrs: [
"10.204.216.17:5269"
],
status: "Up",
type: "XMPP",
name: "control-node:10.204.216.17",
description: "OpenSent"
},
{
server_addrs: [
"10.204.217.53:5269"
],
status: "Up",
type: "XMPP",
name: "control-node:10.204.217.53",
description: "OpenSent"
},
{
server_addrs: [
"10.204.217.53:53"
],
status: "Up",
type: "XMPP",
name: "dns-server:10.204.217.53",
description: "OpenSent"
},
{
server_addrs: [
"10.204.217.60:53"
],
status: "Up",
type: "XMPP",
name: "dns-server:10.204.217.60",
description: "OpenSent"
},
{
server_addrs: [
"10.204.216.17:8086"
],
status: "Up",
type: "Collector",
name: null,
description: "Established"
},
{
server_addrs: [
"10.204.217.53:5998"
],
status: "Up",
type: "Discovery",
name: "Collector",
description: "SubscribeResponse"
},
{
server_addrs: [
"10.204.217.53:5998"
],
status: "Up",
type: "Discovery",
name: "dns-server",
description: "SubscribeResponse"
},
{
server_addrs: [
"10.204.217.53:5998"
],
status: "Up",
type: "Discovery",
name: "xmpp-server",
description: "SubscribeResponse"
}
]
}
],
process_info: [
{
process_name: "contrail-vrouter-agent",
start_count: 2,
process_state: "PROCESS_STATE_RUNNING",
last_stop_time: null,
core_file_list: [ ],
last_start_time: "1467976527710074",
stop_count: 0,
last_exit_time: null,
exit_count: 0
},
{
process_name: "contrail-vrouter-nodemgr",
start_count: 2,
process_state: "PROCESS_STATE_RUNNING",
last_stop_time: null,
core_file_list: [ ],
last_start_time: "1467976527706586",
stop_count: 0,
last_exit_time: null,
exit_count: 0
}
]
},
ContrailConfig: {
deleted: false,
elements: {
display_name: ""nodeh3"",
virtual_router_ip_address: ""10.204.217.107"",
virtual_router_dpdk_enabled: "false",
parent_uuid: ""9cdf12d5-0eb2-46a3-a0e2-49f8c3fceb58"",
parent_href: ""http://0.0.0.0:9100/global-system-config/9cdf12d5-0eb2-46a3-a0e2-49f8c3fceb58"",
parent_type: ""global-system-config"",
perms2: "{"owner": "aedfe72b0e6748d2854b9b6617d1f56b", "owner_access": 7, "global_access": 0, "share": []}",
virtual_router_type: "[]",
fq_name: "["default-global-system-config", "nodeh3"]",
id_perms: "{"enable": true, "uuid": {"uuid_mslong": 3015154081866402783, "uuid_lslong": 11747630932888138302}, "created": "2016-07-08T11:12:30.722556", "description": null, "creator": null, "user_visible": true, "last_modified": "2016-07-08T11:12:30.722556", "permissions": {"owner": "admin", "owner_access": 7, "other_access": 7, "group": "cloud-admin", "group_access": 7}}",
uuid: ""29d7faa9-98b7-4bdf-a307-f8178330323e""
}
},
UVEAlarms: {
alarms: [
{
severity: 4,
ack: false,
timestamp: 1467976450508934,
rules: [ ],
token: "eyJ0aW1lc3RhbXAiOiAxNDY3OTc2NDUwNTA4OTM0LCAiaHR0cF9wb3J0IjogNTk5NSwgImhvc3RfaXAiOiAiMTAuMjA0LjIxNi4xNyJ9",
type: "phyif-bandwidth"
}
]
},
VrouterStatsAgent: {
exception_packets: 40235,
total_in_bandwidth_utilization: 0.0876345,
in_bytes: 0,
cpu_info: {
sys_mem_info: {
cached: 451592,
total: 65683280,
used: 1466476,
free: 64216804,
buffers: 50328
},
num_cpu: 4,
cpu_share: 1.66667,
meminfo: {
virt: 893012,
peakvirt: 893144,
res: 191764
},
cpuload: {
fifteen_min_avg: 0.09,
five_min_avg: 0.1575,
one_min_avg: 0.195
}
},
flow_export_drops: 0,
flow_export_disable_drops: 0,
uptime: 1467976531988791,
out_bps_ewm: {
p2p1: {
sigma: 0.366285,
samples: 33,
stddev: 6416.47,
mean: 31753.7
}
},
phy_band_out_bps: {
p2p1: "34104"
},
phy_if_5min_usage: [
{
in_bandwidth_usage: 859068,
out_bandwidth_usage: 34025,
name: "p2p1"
}
],
xmpp_stats_list: [
{
ip: "10.204.216.17",
reconnects: 1,
in_msgs: 0,
out_msgs: 2
},
{
ip: "10.204.217.53",
reconnects: 1,
in_msgs: 0,
out_msgs: 1
}
],
out_tpkts: 0,
phy_if_stats_list: [
{
out_bytes: 5552667,
name: "p2p1",
duplexity: 1,
out_pkts: 39900,
in_bytes: 57903650,
in_pkts: 102982,
speed: 1000
}
],
exception_packets_dropped: 8,
total_flows: 0,
flow_rate: {
active_flows: 0,
max_flow_deletes_per_second: 0,
added_flows: 0,
deleted_flows: 0,
min_flow_adds_per_second: 0,
min_flow_deletes_per_second: 0,
max_flow_adds_per_second: 0
},
in_bps_ewm: {
p2p1: {
sigma: 0.587285,
samples: 33,
stddev: 307073,
mean: 738575
}
},
out_bytes: 0,
phy_band_in_bps: {
p2p1: "918914"
},
in_tpkts: 0,
vhost_stats: {
out_bytes: 57903650,
name: "vhost0",
duplexity: -1,
out_pkts: 102982,
in_bytes: 5551533,
in_pkts: 39873,
speed: -1
},
total_out_bandwidth_utilization: 0.00325241,
aged_flows: 0,
exception_packets_allowed: 40227,
flow_export_sampling_drops: 0
},
VrouterAgent: {
dns_servers: [
"10.204.217.53",
"10.204.217.60"
],
log_category: null,
control_ip: "10.204.217.107",
control_node_list_cfg: [
"0.0.0.0",
"0.0.0.0"
],
hostname_cfg: "nodeh3",
ds_xs_instances: 3,
log_level: "SYS_NOTICE",
vhost_cfg: {
ip: "10.204.217.107",
ip_prefix_len: 24,
name: "vhost0",
gateway: "10.204.217.254"
},
phy_if: [
{
name: "p2p1",
mac_address: "00:25:90:c8:f3:b6"
}
],
log_local: true,
vhost_if: {
name: "vhost0",
mac_address: "00:25:90:c8:f3:b6"
},
platform: "HOST",
sandesh_http_port: 8085,
eth_name: "p2p1",
log_flow: false,
headless_mode_cfg: false,
self_ip_list: [
"10.204.217.107"
],
config_file: "/etc/contrail/contrail-vrouter-agent.conf",
tunnel_type: "MPLSoGRE",
ll_max_vm_flows_cfg: 2048,
ll_max_system_flows_cfg: 2048,
flow_cache_timeout_cfg: 0,
max_vm_flows_cfg: 100,
dns_server_list_cfg: [
"0.0.0.0",
"0.0.0.0"
],
build_info: "{"build-info":[{"build-time":"2016-07-01 07:17:56.622712","build-hostname":"contrail-ec-build17","build-user":"contrail-builder","build-version":"3.1.0.0","build-id":"3.1.0.0-2739","build-number":"2739"}]}",
hypervisor: "kvm",
ds_addr: "10.204.217.53",
mode: "VROUTER",
xmpp_peer_list: [
{
status: true,
ip: "10.204.216.17",
primary: true,
setup_time: 1467976532982058
},
{
status: true,
ip: "10.204.217.53",
primary: false,
setup_time: 1467976532983079
}
],
log_file: "/var/log/contrail/contrail-vrouter-agent.log"
}
}

[Anish Mehta (amehta00)]

Please provide /var/log/contrail/contrail-alarm-gen-stdout.log and /var/log/contrail/contrail-alarm-gen.log

[Ankit Jain (ankitja)]

contrail-alarm-gen.log
07/11/2016 12:33:39 PM [contrail-alarm-gen]: Exception AttributeError in Alarm Processing. Arguments:
("'NoneType' object has no attribute 'or_list'",) : traceback Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/opserver/alarmgen.py", line 227, in process_alarms
    alarm.config(), uv, local_uve)
  File "/usr/lib/python2.7/dist-packages/opserver/alarmgen.py", line 365, in _evaluate_uve_for_alarms
    for cfg_and_list in alarm_cfg.alarm_rules.or_list:
AttributeError: 'NoneType' object has no attribute 'or_list'

Please look at root@nodeg13:/var/log/contrail# contrail-alarm-gen-stdout.log for detailed logs.

[Anish Mehta (amehta00)]

This change is needed:
https://review.opencontrail.org/#/c/21692/

[Ankit Jain (ankitja)]

This seems to be ok in the latest build 2740.