Description: Backport dict segfault patch from 3.6 There are a few different related segfault issues related to dicts that show up during module loading in heavy use of Ansible under python3.5. They have been fixed upstream in 3.6 and backported upstream to 3.5. . python3.5 (3.5.2-2ubuntu0~16.04.2) xenial; urgency=medium . * Import backported dict segfault patch for http://bugs.python.org/issue27945 Origin: upstream, https://github.com/python/cpython/commit/2f7f533cf6fb57fcedcbc7bd454ac59fbaf2c655 Bug: http://bugs.python.org/issue27945 Forwarded: not-needed Last-Update: 2017-08-18 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -958,6 +958,92 @@ support.check_free_after_iterating(self, lambda d: iter(d.values()), dict) support.check_free_after_iterating(self, lambda d: iter(d.items()), dict) + def test_equal_operator_modifying_operand(self): + # test fix for seg fault reported in issue 27945 part 3. + class X(): + def __del__(self): + dict_b.clear() + + def __eq__(self, other): + dict_a.clear() + return True + + def __hash__(self): + return 13 + + dict_a = {X(): 0} + dict_b = {X(): X()} + self.assertTrue(dict_a == dict_b) + + def test_fromkeys_operator_modifying_dict_operand(self): + # test fix for seg fault reported in issue 27945 part 4a. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1): 1, X(2): 2} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_fromkeys_operator_modifying_set_operand(self): + # test fix for seg fault reported in issue 27945 part 4b. + class X(int): + def __hash__(self): + return 13 + + def __eq__(self, other): + if len(d) > 1: + d.clear() + return False + + d = {} # this is required to exist so that d can be constructed! + d = {X(1), X(2)} + try: + dict.fromkeys(d) # shouldn't crash + except RuntimeError: # implementation defined + pass + + def test_dictitems_contains_use_after_free(self): + class X: + def __eq__(self, other): + d.clear() + return NotImplemented + + d = {0: set()} + (0, X()) in d.items() + + def test_init_use_after_free(self): + class X: + def __hash__(self): + pair[:] = [] + return 13 + + pair = [X(), 123] + dict([pair]) + + def test_oob_indexing_dictiter_iternextitem(self): + class X(int): + def __del__(self): + d.clear() + + d = {i: X(i) for i in range(8)} + + def iter_and_mutate(): + for result in d.items(): + if result[0] == 2: + d[2] = None # free d[2] --> X(2).__del__ was called + + self.assertRaises(RuntimeError, iter_and_mutate) + + from test import mapping_tests class GeneralMappingTests(mapping_tests.BasicTestMappingProtocol): --- a/Misc/ACKS +++ b/Misc/ACKS @@ -524,6 +524,7 @@ Kim Gräsman Nathaniel Gray Eddy De Greef +Duane Griffin Grant Griffin Andrea Griffini Duncan Grisby --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -790,56 +790,61 @@ PyDictKeyEntry *ep; assert(key != dummy); + Py_INCREF(key); + Py_INCREF(value); if (mp->ma_values != NULL && !PyUnicode_CheckExact(key)) { if (insertion_resize(mp) < 0) - return -1; + goto Fail; } ep = mp->ma_keys->dk_lookup(mp, key, hash, &value_addr); - if (ep == NULL) { - return -1; - } + if (ep == NULL) + goto Fail; + assert(PyUnicode_CheckExact(key) || mp->ma_keys->dk_lookup == lookdict); - Py_INCREF(value); MAINTAIN_TRACKING(mp, key, value); old_value = *value_addr; if (old_value != NULL) { assert(ep->me_key != NULL && ep->me_key != dummy); *value_addr = value; Py_DECREF(old_value); /* which **CAN** re-enter (see issue #22653) */ + Py_DECREF(key); } else { if (ep->me_key == NULL) { - Py_INCREF(key); if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ - if (insertion_resize(mp) < 0) { - Py_DECREF(key); - Py_DECREF(value); - return -1; - } + if (insertion_resize(mp) < 0) + goto Fail; ep = find_empty_slot(mp, key, hash, &value_addr); } + mp->ma_used++; + *value_addr = value; mp->ma_keys->dk_usable--; assert(mp->ma_keys->dk_usable >= 0); ep->me_key = key; ep->me_hash = hash; + assert(ep->me_key != NULL && ep->me_key != dummy); } else { + mp->ma_used++; + *value_addr = value; if (ep->me_key == dummy) { - Py_INCREF(key); ep->me_key = key; ep->me_hash = hash; Py_DECREF(dummy); } else { assert(_PyDict_HasSplitTable(mp)); + Py_DECREF(key); } } - mp->ma_used++; - *value_addr = value; - assert(ep->me_key != NULL && ep->me_key != dummy); } return 0; + +Fail: + Py_DECREF(value); + Py_DECREF(key); + return -1; } /* @@ -2012,11 +2017,18 @@ /* Update/merge with this (key, value) pair. */ key = PySequence_Fast_GET_ITEM(fast, 0); value = PySequence_Fast_GET_ITEM(fast, 1); + Py_INCREF(key); + Py_INCREF(value); if (override || PyDict_GetItem(d, key) == NULL) { int status = PyDict_SetItem(d, key, value); - if (status < 0) + if (status < 0) { + Py_DECREF(key); + Py_DECREF(value); goto Fail; + } } + Py_DECREF(key); + Py_DECREF(value); Py_DECREF(fast); Py_DECREF(item); } @@ -2275,14 +2287,15 @@ bval = NULL; else bval = *vaddr; - Py_DECREF(key); if (bval == NULL) { + Py_DECREF(key); Py_DECREF(aval); if (PyErr_Occurred()) return -1; return 0; } cmp = PyObject_RichCompareBool(aval, bval, Py_EQ); + Py_DECREF(key); Py_DECREF(aval); if (cmp <= 0) /* error or not equal */ return cmp; @@ -3106,7 +3119,7 @@ static PyObject *dictiter_iternextitem(dictiterobject *di) { - PyObject *key, *value, *result = di->di_result; + PyObject *key, *value, *result; Py_ssize_t i, mask, offset; PyDictObject *d = di->di_dict; PyObject **value_ptr; @@ -3142,22 +3155,27 @@ if (i > mask) goto fail; - if (result->ob_refcnt == 1) { + di->len--; + key = d->ma_keys->dk_entries[i].me_key; + value = *value_ptr; + Py_INCREF(key); + Py_INCREF(value); + result = di->di_result; + if (Py_REFCNT(result) == 1) { + PyObject *oldkey = PyTuple_GET_ITEM(result, 0); + PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ Py_INCREF(result); - Py_DECREF(PyTuple_GET_ITEM(result, 0)); - Py_DECREF(PyTuple_GET_ITEM(result, 1)); + Py_DECREF(oldkey); + Py_DECREF(oldvalue); } else { result = PyTuple_New(2); if (result == NULL) return NULL; + PyTuple_SET_ITEM(result, 0, key); /* steals reference */ + PyTuple_SET_ITEM(result, 1, value); /* steals reference */ } - di->len--; - key = d->ma_keys->dk_entries[i].me_key; - value = *value_ptr; - Py_INCREF(key); - Py_INCREF(value); - PyTuple_SET_ITEM(result, 0, key); /* steals reference */ - PyTuple_SET_ITEM(result, 1, value); /* steals reference */ return result; fail: @@ -3650,6 +3668,7 @@ static int dictitems_contains(_PyDictViewObject *dv, PyObject *obj) { + int result; PyObject *key, *value, *found; if (dv->dv_dict == NULL) return 0; @@ -3663,7 +3682,10 @@ return -1; return 0; } - return PyObject_RichCompareBool(value, found, Py_EQ); + Py_INCREF(found); + result = PyObject_RichCompareBool(value, found, Py_EQ); + Py_DECREF(found); + return result; } static PySequenceMethods dictitems_as_sequence = {