python-asdf is broken on big endian architectures

Bug #1686079 reported by Dimitri John Ledkov on 2017-04-25
8
This bug affects 1 person
Affects Status Importance Assigned to Milestone
python-asdf (Ubuntu)
High
Unassigned

Bug Description

In ubuntu we execute test suites on all architectures, and the tests are failing on s390x.

You can see the full logs at http://autopkgtest.ubuntu.com/packages/python-asdf/zesty/s390x

I will attach some of them here:

=================================== FAILURES ===================================
__________________________________ test_table __________________________________

tmpdir = local('/tmp/pytest-of-ubuntu/pytest-0/test_table0')

    def test_table(tmpdir):
        table = np.array(
            [(0, 1, (2, 3)), (4, 5, (6, 7))],
            dtype=[(str('MINE'), np.int8),
                   (str(''), np.float64),
                   (str('arr'), '>i4', (2,))])

        tree = {'table_data': table}

        def check_raw_yaml(content):
            tree = yaml.load(
                re.sub(br'!core/\S+', b'', content))

            assert tree['table_data'] == {
                'datatype': [
                    {'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'},
                    {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'},
                    {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}
                    ],
                'shape': [2],
                'source': 0,
                'byteorder': 'big'
                }

> helpers.assert_roundtrip_tree(tree, tmpdir, None, check_raw_yaml)

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:223:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/lib/python3/dist-packages/asdf/tests/helpers.py:131: in assert_roundtrip_tree
    raw_yaml_check_func(content)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

content = b"%YAML 1.1\n%TAG ! tag:stsci.edu:asdf/\n--- !core/asdf-1.0.0\nasdf_library: !core/software-1.0.0 {author: Space Teles...me: f1}\n - byteorder: big\n datatype: int32\n name: arr\n shape: [2]\n byteorder: big\n shape: [2]\n...\n"

    def check_raw_yaml(content):
        tree = yaml.load(
            re.sub(br'!core/\S+', b'', content))

> assert tree['table_data'] == {
            'datatype': [
                {'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'},
                {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'},
                {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}
                ],
            'shape': [2],
            'source': 0,
            'byteorder': 'big'
            }
E assert {'byteorder':..., 'source': 0} == {'byteorder': ..., 'source': 0}
E Omitting 3 identical items, use -v to show
E Differing items:
E {'datatype': [{'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'}, {'byteorder': 'big', 'datatype': 'float64', 'name': 'f1'}, {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}]} != {'datatype': [{'byteorder': 'big', 'datatype': 'int8', 'name': 'MINE'}, {'byteorder': 'little', 'datatype': 'float64', 'name': 'f1'}, {'byteorder': 'big', 'datatype': 'int32', 'name': 'arr', 'shape': [2]}]}
E Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:212: AssertionError
___________________________ test_table_nested_fields ___________________________

tmpdir = local('/tmp/pytest-of-ubuntu/pytest-0/test_table_nested_fields0')

    def test_table_nested_fields(tmpdir):
        table = np.array(
            [(0, (1, 2)), (4, (5, 6)), (7, (8, 9))],
            dtype=[(str('A'), np.int64),
                   (str('B'), [(str('C'), np.int64), (str('D'), np.int64)])])

        tree = {'table_data': table}

        def check_raw_yaml(content):
            tree = yaml.load(
                re.sub(br'!core/\S+', b'', content))

            assert tree['table_data'] == {
                'datatype': [
                    {'datatype': 'int64', 'name': 'A', 'byteorder': 'little'},
                    {'datatype': [
                        {'datatype': 'int64', 'name': 'C', 'byteorder': 'little'},
                        {'datatype': 'int64', 'name': 'D', 'byteorder': 'little'}
                    ], 'name': 'B', 'byteorder': 'big'}],
                'shape': [3],
                'source': 0,
                'byteorder': 'big'
            }

> helpers.assert_roundtrip_tree(tree, tmpdir, None, check_raw_yaml)

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:250:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/usr/lib/python3/dist-packages/asdf/tests/helpers.py:131: in assert_roundtrip_tree
    raw_yaml_check_func(content)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

content = b"%YAML 1.1\n%TAG ! tag:stsci.edu:asdf/\n--- !core/asdf-1.0.0\nasdf_library: !core/software-1.0.0 {author: Space Teles... int64, name: C}\n - {byteorder: big, datatype: int64, name: D}\n name: B\n byteorder: big\n shape: [3]\n...\n"

    def check_raw_yaml(content):
        tree = yaml.load(
            re.sub(br'!core/\S+', b'', content))

> assert tree['table_data'] == {
            'datatype': [
                {'datatype': 'int64', 'name': 'A', 'byteorder': 'little'},
                {'datatype': [
                    {'datatype': 'int64', 'name': 'C', 'byteorder': 'little'},
                    {'datatype': 'int64', 'name': 'D', 'byteorder': 'little'}
                ], 'name': 'B', 'byteorder': 'big'}],
            'shape': [3],
            'source': 0,
            'byteorder': 'big'
        }
E assert {'byteorder':..., 'source': 0} == {'byteorder': ..., 'source': 0}
E Omitting 3 identical items, use -v to show
E Differing items:
E {'datatype': [{'byteorder': 'big', 'datatype': 'int64', 'name': 'A'}, {'byteorder': 'big', 'datatype': [{'byteorder': 'big', 'datatype': 'int64', 'name': 'C'}, {'byteorder': 'big', 'datatype': 'int64', 'name': 'D'}], 'name': 'B'}]} != {'datatype': [{'byteorder': 'little', 'datatype': 'int64', 'name': 'A'}, {'byteorder': 'big', 'datatype': [{'byteorder': 'little', 'datatype': 'int64', 'name': 'C'}, {'byteorder': 'little', 'datatype': 'int64', 'name': 'D'}], 'name': 'B'}]}
E Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tags/core/tests/test_ndarray.py:238: AssertionError
________________________________ test_checksum _________________________________

tmpdir = '/tmp/pytest-of-ubuntu/pytest-0/test_checksum0'

    def test_checksum(tmpdir):
        tmpdir = str(tmpdir)
        path = os.path.join(tmpdir, 'test.asdf')

        my_array = np.arange(0, 64, dtype=np.int64).reshape((8, 8))
        tree = {'my_array': my_array}
        ff = asdf.AsdfFile(tree)
        ff.write_to(path)

        with asdf.AsdfFile.open(path, validate_checksums=True) as ff:
            assert type(ff.blocks._internal_blocks[0].checksum) == bytes
> assert ff.blocks._internal_blocks[0].checksum == \
                b'\xcaM\\\xb8t_L|\x00\n+\x01\xf1\xcfP1'
E assert b'\xab\xc17\x...x97\x1b@\x86G' == b'\xcaM\\\xb8t...x01\xf1\xcfP1'
E At index 0 diff: 171 != 202
E Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tests/test_low_level.py:740: AssertionError
_____________________________ test_checksum_update _____________________________

tmpdir = '/tmp/pytest-of-ubuntu/pytest-0/test_checksum_update0'

    def test_checksum_update(tmpdir):
        tmpdir = str(tmpdir)
        path = os.path.join(tmpdir, 'test.asdf')

        my_array = np.arange(0, 64, dtype=np.int64).reshape((8, 8))

        tree = {'my_array': my_array}
        ff = asdf.AsdfFile(tree)
        ff.write_to(path)

        with asdf.AsdfFile.open(path, mode='rw') as ff:
            ff.tree['my_array'][7, 7] = 0.0
            # update() should update the checksum, even if the data itself
            # is memmapped and isn't expressly re-written.
            ff.update()

        with asdf.AsdfFile.open(path, validate_checksums=True) as ff:
> assert ff.blocks._internal_blocks[0].checksum == \
                b'T\xaf~[\x90\x8a\x88^\xc2B\x96D,N\xadL'
E assert b'\x01\xf5\xb...x99`\xd7\xa9w' == b'T\xaf~[\x90\...B\x96D,N\xadL'
E At index 0 diff: 1 != 84
E Use -v to get the full diff

/usr/lib/python3/dist-packages/asdf/tests/test_low_level.py:761: AssertionError

Dimitri John Ledkov (xnox) wrote :
Changed in python-asdf (Ubuntu):
status: New → Confirmed
importance: Undecided → High
To post a comment you must log in.
This report contains Public information  Edit
Everyone can see this information.

Other bug subscribers

Remote bug watches

Bug watches keep track of this bug in other bug trackers.