Build:
======
Contrail 2.21-14
Logs:
======
Logs are stored here :
jdeepak@ttsv-shell04 /volume/vmcores/sagrawal/jdeepak_temp/NFV/contrail_bug/1537844
$ ls
total 248872
-rwxrwxrwx 1 jdeepak software 253839360 Jan 25 11:10 logs_PR-1537844.tar
jdeepak@ttsv-shell04 /volume/vmcores/sagrawal/jdeepak_temp/NFV/contrail_bug/1537844
Problem:
========
Multi node contrail setup , with 3 servers as contrail config, control and db .
Instantiating 30 instances parallely, one one occassion saw 30 stacks being created (CREATE_COMPLETE) but 5 instances are stuck in BUILD (nova)
root@ccra-15:~# heat stack-list
n+--------------------------------------+------------------------------------------+-----------------+----------------------+
| id | stack_name | stack_status | creation_time |
+--------------------------------------+------------------------------------------+-----------------+----------------------+
| 40618fd7-ae72-4496-9837-4f6f83a04a0d | jsm-4275e3bd-3e69-4693-b9b0-0fc7d6429271 | UPDATE_COMPLETE | 2016-01-25T17:24:46Z |
| 6b34fbbc-c50e-4a1a-a018-47c57caeb7a9 | jsm-a74126fb-5212-4576-bfdb-c5d093c09bd0 | CREATE_COMPLETE | 2016-01-25T17:24:57Z |
| 279e69b3-7b2b-40ba-8f69-c6f8a5f45253 | jsm-2cacd937-0603-4874-8e9a-b69783a6b1db | CREATE_COMPLETE | 2016-01-25T17:25:03Z |
| c24060e9-6f2a-47ae-a59b-989578553a55 | jsm-18e52e04-2466-4a68-895a-00f62a0187cb | CREATE_COMPLETE | 2016-01-25T17:25:05Z |
| af6e05f5-d760-4744-9c90-622201559332 | jsm-ed90f47b-cc94-43de-88df-5b345d764914 | UPDATE_COMPLETE | 2016-01-25T17:25:16Z |
| 814a8ae2-43bf-48e7-917f-cb9e4c1aa811 | jsm-990daca4-3096-438b-877c-36c7b058318d | UPDATE_COMPLETE | 2016-01-25T17:25:19Z |
| 8a026d5c-910c-443a-bc40-9230aa483624 | jsm-7b4a695a-93b5-4d87-b843-47d0f21f9a1e | UPDATE_COMPLETE | 2016-01-25T17:25:39Z |
| 84829220-961f-488a-8b2c-7aaedfcf6412 | jsm-ee2c5a48-4835-42b2-ae35-f5ab607a8886 | UPDATE_COMPLETE | 2016-01-25T17:25:44Z |
| 9d22c359-84ec-4a3a-8c03-b9cb922e9fb5 | jsm-5fdcd561-52cd-4164-b46a-c925413cf9e4 | CREATE_COMPLETE | 2016-01-25T17:25:48Z |
| 8fb3721a-3583-4035-9930-12be33fc29fe | jsm-ad3d497c-7da3-4007-8f45-32f4b836c31f | CREATE_COMPLETE | 2016-01-25T17:25:49Z |
| df517050-dad9-4e0e-bc7c-c9c21c923127 | jsm-730da1cb-84c6-4410-a404-db9763248b51 | UPDATE_COMPLETE | 2016-01-25T17:25:57Z |
| 84e881e2-a4ec-4e51-95b7-506caf9a2e3b | jsm-91fc8268-60c1-4fd5-95e7-a08f91c0ab22 | UPDATE_COMPLETE | 2016-01-25T17:25:58Z |
| befd7c37-e2b8-45e0-aff7-e8a9b34a76dd | jsm-065c814a-4d93-45e0-b582-dd2031026aad | UPDATE_COMPLETE | 2016-01-25T17:26:00Z |
| 86a9f37c-d486-48b7-a40a-27f571f188f9 | jsm-91c8542b-0b5a-4a6b-88fc-d9c0c26ca249 | UPDATE_COMPLETE | 2016-01-25T17:26:03Z |
| 5ebdde79-9ad7-498a-8032-f5ae19ece832 | jsm-e7a8b082-fd5d-478a-be5b-d061a621e8b3 | UPDATE_COMPLETE | 2016-01-25T17:26:06Z |
| 427db1dd-f4c9-4df4-978a-d0823132a655 | jsm-875d84b8-e2d4-4db9-8abd-e577137a6575 | UPDATE_COMPLETE | 2016-01-25T17:26:10Z |
| 6bd2e219-3a59-421a-9786-5060052ec76e | jsm-1917d625-7c3e-43df-b13f-5840584fa328 | UPDATE_COMPLETE | 2016-01-25T17:26:13Z |
| 40142c32-50f8-453f-8417-80223cd21d31 | jsm-ca730fc0-0f3e-4b41-a83c-11ff4a06efb7 | UPDATE_COMPLETE | 2016-01-25T17:26:18Z |
| cdd2b110-d1db-44a1-9d54-9bd6e86b7920 | jsm-7ad8a06f-feb8-4538-a613-bc468b7ed8a8 | UPDATE_COMPLETE | 2016-01-25T17:26:23Z |
| 050f7f9f-9ed8-44a1-8402-39ba07dba039 | jsm-08de2de6-86d2-4bbf-9fb7-8c2baf463614 | UPDATE_COMPLETE | 2016-01-25T17:26:27Z |
| c5f2bf8b-63ec-4f5f-bee7-88ff5108bd41 | jsm-8031e872-2a9a-49c5-8464-d4784a142e3b | UPDATE_COMPLETE | 2016-01-25T17:26:27Z |
| 3b49144d-19f7-45cc-9609-a5fe29364d7c | jsm-9dbc6190-34b1-4289-bec7-e19aaeb35537 | UPDATE_COMPLETE | 2016-01-25T17:26:28Z |
| 5266000f-4eea-465c-a1ce-353b564c6a19 | jsm-eab2ce1f-6fa5-49c4-b98b-02d0b7bc5b1f | UPDATE_COMPLETE | 2016-01-25T17:26:29Z |
| c7c4741a-ad3b-4357-b211-07ad1b2498ce | jsm-fcb71465-e26c-4247-9472-f762340a0fd6 | UPDATE_COMPLETE | 2016-01-25T17:26:29Z |
| 9df3fadf-3db3-49bd-b88a-317b95a756b3 | jsm-e1bd6695-704b-419f-ab70-6fee3cb7ecae | UPDATE_COMPLETE | 2016-01-25T17:26:35Z |
| 226f82d1-edd4-4aa6-abef-64038a72ee69 | jsm-8f00c5f5-c6c4-41b4-88d2-0af71027b051 | UPDATE_COMPLETE | 2016-01-25T17:26:36Z |
| a5093bbd-7032-4d96-935c-cab7eaa5f598 | jsm-9b307579-1cb9-4662-bc40-a42612519f63 | UPDATE_COMPLETE | 2016-01-25T17:26:39Z |
| cbd2e81b-7502-4356-9fb0-f9032863fd6e | jsm-5e5d004f-fc0b-4b0c-9d87-a898b9a708f8 | UPDATE_COMPLETE | 2016-01-25T17:26:40Z |
| d016532b-d298-491c-84db-f9f5ded7df76 | jsm-7479df63-8152-4951-9797-33f4c76e1319 | UPDATE_COMPLETE | 2016-01-25T17:26:41Z |
| c243f1c9-713e-493d-ba9e-aa5a8bbf9e06 | jsm-915e1425-2b78-4de0-99ad-4724170e345f | UPDATE_COMPLETE | 2016-01-25T17:26:47Z |
+--------------------------------------+------------------------------------------+-----------------+----------------------+
root@ccra-15:~# nova list | grep ACTIVE | wc -l
25
root@ccra-15:~# nova list | grep BUILD | wc -l
5
