diff -Nru neutron-2014.1.5/debian/changelog neutron-2014.1.5/debian/changelog --- neutron-2014.1.5/debian/changelog 2015-06-24 23:04:10.000000000 +0800 +++ neutron-2014.1.5/debian/changelog 2015-12-17 16:37:09.000000000 +0800 @@ -1,3 +1,11 @@ +neutron (1:2014.1.5-0ubuntu2) trusty; urgency=medium + + * Backport upstream release. (LP: #1318721): + - d/p/fix-reconnect-race-condition-with-rabbitmq-cluster.patch: + Redeclare if exception is catched after self.queue.declare() failed. + + -- Hui Xiang Thu, 17 Dec 2015 16:35:40 +0800 + neutron (1:2014.1.5-0ubuntu1) trusty; urgency=medium * Resynchronize with stable/icehouse (877df58) (LP: #1467533): diff -Nru neutron-2014.1.5/debian/patches/fix-reconnect-race-condition-with-rabbitmq-cluster.patch neutron-2014.1.5/debian/patches/fix-reconnect-race-condition-with-rabbitmq-cluster.patch --- neutron-2014.1.5/debian/patches/fix-reconnect-race-condition-with-rabbitmq-cluster.patch 1970-01-01 08:00:00.000000000 +0800 +++ neutron-2014.1.5/debian/patches/fix-reconnect-race-condition-with-rabbitmq-cluster.patch 2016-01-07 15:15:26.000000000 +0800 @@ -0,0 +1,74 @@ +Description: Fix reconnect race condition with RabbitMQ cluster + + Retry declaring Queue if it fail to workaround the race condition + that may happen when both Neutron and RabbitMQ cluster are trying + to create and delete (Respectively) Queues and Exchanges that have + auto-delete flag set. + + Change-Id: I90febd7c8358b7cccf852c272810e2fad3d91dcd + Closes-Bug: #1318721 + +Author: Mouad Benchchaoui +Origin: backport, https://review.openstack.org/#/c/93399/1 +Bug: https://bugs.launchpad.net/neutron/+bug/1318721 +--- +This patch header follows DEP-3: http://dep.debian.net/deps/dep3/ +Index: neutron-2014.1.5/neutron/openstack/common/rpc/impl_kombu.py +=================================================================== +--- neutron-2014.1.5.orig/neutron/openstack/common/rpc/impl_kombu.py 2015-06-18 22:26:11.000000000 +0000 ++++ neutron-2014.1.5/neutron/openstack/common/rpc/impl_kombu.py 2015-12-01 02:40:27.806902000 +0000 +@@ -137,7 +137,53 @@ + self.channel = channel + self.kwargs['channel'] = channel + self.queue = kombu.entity.Queue(**self.kwargs) +- self.queue.declare() ++ try: ++ self.queue.declare() ++ except Exception: ++ # NOTE(Mouad): Catching Exception b/c Kombu doesn't raise a proper ++ # error instead it's raise what the underlying transport library ++ # raise which can be either 'ampq.NotFound' or ++ # 'librabbitmq.ChannelError' depending on which transport library ++ # is used. ++ LOG.exception("Declaring queue fail retrying ...") ++ # NOTE(Mouad): We need to re-try Queue creation in case the Queue ++ # was created with auto-delete this instruct the Broker to delete ++ # the Queue when the last Consumer disconnect from it, and the ++ # Exchange when the last Queue is deleted from this Exchange. ++ # ++ # Now in a RabbitMQ cluster setup, if the cluster node that we are ++ # connected to go down, 2 things will happen: ++ # ++ # 1. From RabbitMQ side the Queues will be deleted from the other ++ # cluster nodes and then the correspanding Exchanges will also ++ # be deleted. ++ # 2. From Neutron side, Neutron will reconnect to another cluster ++ # node and start creating Exchanges then Queues then Binding ++ # (The order is important to understand the problem). ++ # ++ # Now this may lead to a race condition, specially if Neutron create ++ # Exchange and Queue and before Neutron could bind Queue to Exchange ++ # RabbitMQ nodes just received the 'signal' that they need to delete ++ # Exchanges with auto-delete that belong to the down node, so they ++ # will delete the Exchanges that was just created, and when Neutron ++ # try to bind Queue with Exchange, the Binding will fail b/c the ++ # Exchange is not found. ++ # ++ # But if the first Queue declartion work and binding was created we ++ # suppose that RabbitMQ will not try to delete the Exchange even if ++ # the auto-delete propagation wasn't received yet, b/c the Queue ++ # have a Consumer now and a Binding exist with the Exchange. ++ # ++ # Note: AMQP 0-9-1 deprecated the use of 'auto-delete' for Exchanges ++ # but according to here[1] RabbitMQ doesn't seem that he will delete ++ # it: ++ # ++ # The 'auto-delete' flag on 'exchange.declare' got deprecated in ++ # 0-9-1. Auto-delete exchanges are actually quite useful, so this ++ # flag should be restored. ++ # ++ # [1] http://www.rabbitmq.com/amqp-0-9-1-errata.html ++ self.queue.declare() + + def _callback_handler(self, message, callback): + """Call callback with deserialized message. diff -Nru neutron-2014.1.5/debian/patches/series neutron-2014.1.5/debian/patches/series --- neutron-2014.1.5/debian/patches/series 2015-06-24 23:04:10.000000000 +0800 +++ neutron-2014.1.5/debian/patches/series 2015-12-17 16:35:01.000000000 +0800 @@ -2,3 +2,4 @@ disable-udev-tests.patch skip-ipv6-tests.patch use-concurrency.patch +fix-reconnect-race-condition-with-rabbitmq-cluster.patch