To pro-actively see what else was broken I did this. The errors below are just 'arch=amd64 release=yakkety'. I'll go ahead and run a full pass.
$ bzr branch lp:simplestreams simplestreams $ cd simplestreams
# I then modified sstream-mirror to only warn on errors # and log to file /tmp/errors.log $ bzr diff === modified file 'simplestreams/mirrors/__init__.py' --- simplestreams/mirrors/__init__.py 2016-06-10 11:20:18 +0000 +++ simplestreams/mirrors/__init__.py 2017-03-01 13:41:33 +0000 @@ -473,9 +473,16 @@ if not self.config.get('item_download', True): return LOG.debug("inserting %s to %s", contentsource.url, data['path']) - self.store.insert(data['path'], contentsource, - checksums=checksum_util.item_checksums(data), - mutable=False, size=data.get('size')) + try: + self.store.insert(data['path'], contentsource, + checksums=checksum_util.item_checksums(data), + mutable=False, size=data.get('size')) + except checksum_util.InvalidChecksum as e: + with open("/tmp/errors.log", "a") as fp: + fp.write("%s\n" % e) + import sys + sys.stderr.write("%s\n" % e) + self._inc_rc(data['path'], src, pedigree)
def insert_index_entry(self, data, src, pedigree, contentsource):
$ url=http://cloud-images.ubuntu.com/daily/streams/v1/index.json $ rm -Rf out.d $ ./tools/tenv sstream-mirror --max=1 $url --progress out.d release=yakkety arch=amd64 ... cat /tmp/errors.log Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64-lxd.tar.xz. Found fd9f97481a3d21f6cf017b5e8b199c7b5a9fbd3fe0896fa697edd44267b93036. Expected 522ea193e4414cc41baa578e82f4eba7b6c5e516049656ba8cdae84a4d10b0cb. read 848 bytes expected 840 bytes. (size 848 expected 840) Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64.manifest. Found a7018e21f0b9119231d7c25cacb360b753d4d40e11bf6fdbda9398ea8e00fc60. Expected d6f2d40732308b2f0a95d2b36fec0d2899976f459536abedfeb5e2af48474326. read 14255 bytes expected 14255 bytes. (size 14255 expected 14255) Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64.squashfs.manifest. Found 0bda9b9a517cb85dd818136d4cef13b998607fc555bc2f6e0ce0637950dddd15. Expected 955f530579667721260241d9c82df219ecb66817ea6aeba70515a5d5033ada92. read 13736 bytes expected 13736 bytes. (size 13736 expected 13736) Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64.img. Found 9a209e74abceb899c73e678758401eed18832e90347ad81f56975525cd718224. Expected ea089b13a75cc6ad92aa6533796f912adf3fd8f860b0134752bade9b6644ccc8. read 342630400 bytes expected 342622208 bytes. (size 342630400 expected 342622208) Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64.tar.gz. Found ebaecba0a4aba918637e7d5ab4861290fd513ec3113b7f2c6be19d2c82382aba. Expected 350f1aebe32a370c16078977e4ee1aa56a97e6c0c3498ced5c9ef8168a127468. read 266919813 bytes expected 266936375 bytes. (size 266919813 expected 266936375) Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64.squashfs. Found 5f39a83a9ae0286a13ca405a59eacc47e8e27abe4806a192944c33945e883b24. Expected a96a84cefac3a3e2cba92cb80a8e49c0d4bf91d8141f3fa0612ee71f3bf2e3d1. read 163205120 bytes expected 163196928 bytes. (size 163205120 expected 163196928) Invalid sha256 Checksum at http://cloud-images.ubuntu.com/daily/server/yakkety/20170222.1/yakkety-server-cloudimg-amd64.ova. Found 4eca9179cd629ca71256153164386215872b8056a90b23d736146ee6d61e8522. Expected d7fa2eb5596842650c50b06b8c1f3406dc52d809891f98e7bbacdc5fce3cf737. read 327761920 bytes expected 327761920 bytes. (size 327761920 expected 327761920)
To pro-actively see what else was broken I did this. The errors below are just 'arch=amd64 release=yakkety'. I'll go ahead and run a full pass.
$ bzr branch lp:simplestreams simplestreams
$ cd simplestreams
# I then modified sstream-mirror to only warn on errors mirrors/ __init_ _.py' mirrors/ __init_ _.py 2016-06-10 11:20:18 +0000 mirrors/ __init_ _.py 2017-03-01 13:41:33 +0000 get('item_ download' , True):
LOG.debug( "inserting %s to %s", contentsource.url, data['path']) insert( data['path' ], contentsource, checksum_ util.item_ checksums( data), get('size' )) insert( data['path' ], contentsource, checksum_ util.item_ checksums( data), get('size' )) util.InvalidChe cksum as e: tmp/errors. log", "a") as fp: write(" %s\n" % e)
self. _inc_rc( data['path' ], src, pedigree)
# and log to file /tmp/errors.log
$ bzr diff
=== modified file 'simplestreams/
--- simplestreams/
+++ simplestreams/
@@ -473,9 +473,16 @@
if not self.config.
return
- self.store.
- checksums=
- mutable=False, size=data.
+ try:
+ self.store.
+ checksums=
+ mutable=False, size=data.
+ except checksum_
+ with open("/
+ fp.write("%s\n" % e)
+ import sys
+ sys.stderr.
+
def insert_ index_entry( self, data, src, pedigree, contentsource):
$ url=http:// cloud-images. ubuntu. com/daily/ streams/ v1/index. json cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64-lxd. tar.xz. Found fd9f97481a3d21f 6cf017b5e8b199c 7b5a9fbd3fe0896 fa697edd44267b9 3036. Expected 522ea193e4414cc 41baa578e82f4eb a7b6c5e51604965 6ba8cdae84a4d10 b0cb. read 848 bytes expected 840 bytes. (size 848 expected 840) cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64.manifest. Found a7018e21f0b9119 231d7c25cacb360 b753d4d40e11bf6 fdbda9398ea8e00 fc60. Expected d6f2d40732308b2 f0a95d2b36fec0d 2899976f459536a bedfeb5e2af4847 4326. read 14255 bytes expected 14255 bytes. (size 14255 expected 14255) cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64.squashfs. manifest. Found 0bda9b9a517cb85 dd818136d4cef13 b998607fc555bc2 f6e0ce0637950dd dd15. Expected 955f53057966772 1260241d9c82df2 19ecb66817ea6ae ba70515a5d5033a da92. read 13736 bytes expected 13736 bytes. (size 13736 expected 13736) cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64.img. Found 9a209e74abceb89 9c73e678758401e ed18832e90347ad 81f56975525cd71 8224. Expected ea089b13a75cc6a d92aa6533796f91 2adf3fd8f860b01 34752bade9b6644 ccc8. read 342630400 bytes expected 342622208 bytes. (size 342630400 expected 342622208) cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64.tar. gz. Found ebaecba0a4aba91 8637e7d5ab48612 90fd513ec3113b7 f2c6be19d2c8238 2aba. Expected 350f1aebe32a370 c16078977e4ee1a a56a97e6c0c3498 ced5c9ef8168a12 7468. read 266919813 bytes expected 266936375 bytes. (size 266919813 expected 266936375) cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64.squashfs. Found 5f39a83a9ae0286 a13ca405a59eacc 47e8e27abe4806a 192944c33945e88 3b24. Expected a96a84cefac3a3e 2cba92cb80a8e49 c0d4bf91d8141f3 fa0612ee71f3bf2 e3d1. read 163205120 bytes expected 163196928 bytes. (size 163205120 expected 163196928) cloud-images. ubuntu. com/daily/ server/ yakkety/ 20170222. 1/yakkety- server- cloudimg- amd64.ova. Found 4eca9179cd629ca 712561531643862 15872b8056a90b2 3d736146ee6d61e 8522. Expected d7fa2eb55968426 50c50b06b8c1f34 06dc52d809891f9 8e7bbacdc5fce3c f737. read 327761920 bytes expected 327761920 bytes. (size 327761920 expected 327761920)
$ rm -Rf out.d
$ ./tools/tenv sstream-mirror --max=1 $url --progress out.d release=yakkety arch=amd64
...
cat /tmp/errors.log
Invalid sha256 Checksum at http://
Invalid sha256 Checksum at http://
Invalid sha256 Checksum at http://
Invalid sha256 Checksum at http://
Invalid sha256 Checksum at http://
Invalid sha256 Checksum at http://
Invalid sha256 Checksum at http://