Contrail 2.21-14 - 2 instances created for one heat stack under scaled scenario

[Deepak Jeyaraman]

Trying to scale upto 1000 instances of iptables VNF (ubuntu 12.02) on a 11 compute node setup.

401st instance has caused this behavior

root@ccra-06:~# contrail-version
Package Version Build-ID | Repo | Package Name
-------------------------------------- ------------------------------ ----------------------------------
contrail-analytics 2.21.1-14 14
contrail-config 2.21.1-14 14
contrail-config-openstack 2.21.1-14 14
contrail-control 2.21.1-14 14
contrail-dns 2.21.1-14 14
contrail-f5 2.21.1-14 14
contrail-fabric-utils 2.21.1-14 14
contrail-heat 2.21.1-14 14
contrail-install-packages 2.21.1-14~icehouse 14
contrail-lib 2.21.1-14 14
contrail-nodemgr 2.21.1-14 14
contrail-nova-networkapi 2.21.1-14 14
contrail-openstack 2.21.1-14 14

root@ccra-06:~# nova list | grep 541bb
| 95c7a1ae-e0d2-4fdf-a821-1c77d2b8ea6c | default-domain__admin__541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-instance__1 | ACTIVE | - | Running | right-net=192.169.1.140; mgmt-net=10.102.81.138; 541bb65b-0c65-47bc-b2bd-c32252e90a05-left-net=17.20.152.3 |
| c3a09b96-6f73-4896-aa6c-24916a5b9b56 | default-domain__admin__541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-instance__1 | ACTIVE | - | Running | right-net=192.169.1.140; mgmt-net=10.102.81.138; 541bb65b-0c65-47bc-b2bd-c32252e90a05-left-net=17.20.152.3 |

root@ccra-06:~# heat stack-list | wc -l
529

root@ccra-06:~# heat stack-show 89312228-d202-4d73-92cf-392e7f93ef6d
+----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+
| Property | Value |
+----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+
| capabilities | [] |
| creation_time | 2015-12-03T03:13:00Z |
| description | HOT template to create network service |
| disable_rollback | True |
| id | 89312228-d202-4d73-92cf-392e7f93ef6d |
| links | http://10.102.28.59:8004/v1/9e20cb419c194e2283b364e14bbc4f2d/stacks/jsm-541bb65b-0c65-47bc-b2bd-c32252e90a05/89312228-d202-4d73-92cf-392e7f93ef6d |
| notification_topics | [] |
| outputs | [ |
| | { |
| | "output_value": [ |
| | "64512:992" |
| | ], |
| | "description": "Route targets of the virtual network", |
| | "output_key": "left-net_route_targets" |
| | }, |
| | { |
| | "output_value": "17.20.152.0/24", |
| | "description": "Subnet CIDR", |
| | "output_key": "left-subnet_cidr" |
| | }, |
| | { |
| | "output_value": [], |
| | "description": "Subnet host routes", |
| | "output_key": "left-subnet_host_routes" |
| | }, |
| | { |
| | "output_value": 1, |
| | "description": "Service instance active service vms", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_active_service_vms" |
| | }, |
| | { |
| | "output_value": "default-domain:admin:541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-instance", |
| | "description": "Service instance FQ name", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_fq_name" |
| | }, |
| | { |
| | "output_value": "17.20.152.1", |
| | "description": "Subnet Gateway IP", |
| | "output_key": "left-subnet_gateway_ip" |
| | }, |
| | { |
| | "output_value": "541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-template", |
| | "description": "Service template name", |
| | "output_key": "LxCIPtable_NAT-0-vnf-template_name" |
| | }, |
| | { |
| | "output_value": [ |
| | { |
| | "name": "default-domain:admin:541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-instance", |
| | "uuid": "ad157a34-f201-46bb-8a1e-961d6c8c01a5" |
| | } |
| | ], |
| | "description": "Service template service instances", |
| | "output_key": "LxCIPtable_NAT-0-vnf-template_service_instances" |
| | }, |
| | { |
| | "output_value": [ |
| | { |
| | "status": "ACTIVE", |
| | "vm_id": "c3a09b96-6f73-4896-aa6c-24916a5b9b56", |
| | "name": "default-domain__admin__541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-instance__1" |
| | } |
| | ], |
| | "description": "Service instance virtual machines", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_virtual_machines" |
| | }, |
| | { |
| | "output_value": null, |
| | "description": "Is virtual network external?", |
| | "output_key": "left-net_external" |
| | }, |
| | { |
| | "output_value": 4, |
| | "description": "Subnet IP version", |
| | "output_key": "left-subnet_ip_version" |
| | }, |
| | { |
| | "output_value": "jsm-541bb65b-0c65-47bc-b2bd-c32252e90a05-left-subnet-5xo63crthojg", |
| | "description": "Subnet name", |
| | "output_key": "left-subnet_name" |
| | }, |
| | { |
| | "output_value": true, |
| | "description": "Subnet DHCP enable", |
| | "output_key": "left-subnet_enable_dhcp" |
| | }, |
| | { |
| | "output_value": "541bb65b-0c65-47bc-b2bd-c32252e90a05-1afc81a2-1878-4859-a579-57bcd8135703-site-access-policy", |
| | "description": "Policy name", |
| | "output_key": "site-access-policy_name" |
| | }, |
| | { |
| | "output_value": "541bb65b-0c65-47bc-b2bd-c32252e90a05-left-net", |
| | "description": "Name of the virtual network", |
| | "output_key": "left-net_name" |
| | }, |
| | { |
| | "output_value": "1b825414-aa8f-4689-90fc-7b3de4138032", |
| | "description": "Subnet network ID", |
| | "output_key": "left-subnet_network_id" |
| | }, |
| | { |
| | "output_value": "default-domain:541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-template", |
| | "description": "Service instance service template", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_service_template" |
| | }, |
| | { |
| | "output_value": "default-domain:admin:541bb65b-0c65-47bc-b2bd-c32252e90a05-left-net", |
| | "description": "FQ name of the virtual network", |
| | "output_key": "left-net_fq_name" |
| | }, |
| | { |
| | "output_value": "9e20cb41-9c19-4e22-83b3-64e14bbc4f2d", |
| | "description": "Service instance tenant ID", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_tenant_id" |
| | }, |
| | { |
| | "output_value": null, |
| | "description": "Is virtual network shared?", |
| | "output_key": "left-net_shared" |
| | }, |
| | { |
| | "output_value": "default-domain:541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-template", |
| | "description": "Service template FQ name", |
| | "output_key": "LxCIPtable_NAT-0-vnf-template_fq_name" |
| | }, |
| | { |
| | "output_value": [ |
| | { |
| | "start": "17.20.152.2", |
| | "end": "17.20.152.254" |
| | } |
| | ], |
| | "description": "Subnet allocation pools", |
| | "output_key": "left-subnet_allocation_pools" |
| | }, |
| | { |
| | "output_value": "9e20cb419c194e2283b364e14bbc4f2d", |
| | "description": "Subnet tenant ID", |
| | "output_key": "left-subnet_tenant_id" |
| | }, |
| | { |
| | "output_value": "default-domain:admin:541bb65b-0c65-47bc-b2bd-c32252e90a05-1afc81a2-1878-4859-a579-57bcd8135703-site-access-policy", |
| | "description": "Policy FQ name", |
| | "output_key": "site-access-policy_fq_name" |
| | }, |
| | { |
| | "output_value": [], |
| | "description": "Subnet DNS name servers", |
| | "output_key": "left-subnet_dns_nameservers" |
| | }, |
| | { |
| | "output_value": "541bb65b-0c65-47bc-b2bd-c32252e90a05-LxCIPtable_NAT-0-vnf-instance", |
| | "description": "Service instance name", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_name" |
| | }, |
| | { |
| | "output_value": "ACTIVE", |
| | "description": "Service instance status", |
| | "output_key": "LxCIPtable_NAT-0-vnf-instance_status" |
| | } |
| | ] |
| parameters | { |
| | "right-net-id": "641bec47-9e5b-462c-a458-872508071245", |
| | "OS::stack_name": "jsm-541bb65b-0c65-47bc-b2bd-c32252e90a05", |
| | "nsid": "541bb65b-0c65-47bc-b2bd-c32252e90a05", |
| | "mgmt-net-id": "bdfbd02d-f75c-4bb4-8c04-1f251f9d3d44", |
| | "OS::stack_id": "89312228-d202-4d73-92cf-392e7f93ef6d", |
| | "left-net-rt": "[u'64512:992']", |
| | "left-subnet-cidr": "17.20.152.0/24" |
| | } |
| stack_name | jsm-541bb65b-0c65-47bc-b2bd-c32252e90a05 |
| stack_status | UPDATE_COMPLETE |
| stack_status_reason | Stack successfully updated |
| template_description | HOT template to create network service |
| timeout_mins | None |
| updated_time | 2015-12-03T03:56:12Z |
+----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------+