ValueError: invalid literal for int() with base 10: 'None' in _parse_index_sizes from broken repository pack-names

[David Caro]

I'm trying to develop a little repo manager like gitosis for git. And when doing the first push from the client to the server, this error shows up:

bzr: ERROR: bzrlib.errors.ErrorFromSmartServer: Error received from smart server: ('error', "invalid literal for int() with base 10: 'None'")

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 920, in exception_to_return_code
    return the_callable(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 1131, in run_bzr
    ret = run(*run_argv)
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 673, in run_argv_aliases
    return self.run(**all_cmd_args)
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 695, in run
    return self._operation.run_simple(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 136, in run_simple
    self.cleanups, self.func, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 166, in _do_with_cleanups
    result = func(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/builtins.py", line 1379, in run
    use_existing_dir=use_existing_dir, no_tree=no_tree)
  File "/usr/lib/python2.7/dist-packages/bzrlib/push.py", line 145, in _show_push_branch
    remember, create_prefix)
  File "/usr/lib/python2.7/dist-packages/bzrlib/controldir.py", line 425, in push_branch
    overwrite, stop_revision=revision_id)
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 1142, in push
    lossy, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3507, in push
    _override_hook_source_branch=_override_hook_source_branch)
  File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 132, in run
    self.cleanups, self.func, self, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 166, in _do_with_cleanups
    result = func(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3562, in _push_with_bound_branches
    result = self._basic_push(overwrite, stop_revision)
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3524, in _basic_push
    graph=graph)
  File "/usr/lib/python2.7/dist-packages/bzrlib/decorators.py", line 218, in write_locked
    result = unbound(self, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3425, in _update_revisions
    self.fetch(stop_revision=stop_revision)
  File "/usr/lib/python2.7/dist-packages/bzrlib/decorators.py", line 218, in write_locked
    result = unbound(self, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3402, in fetch
    fetch_spec=fetch_spec)
  File "/usr/lib/python2.7/dist-packages/bzrlib/remote.py", line 2157, in fetch
    find_ghosts=find_ghosts, fetch_spec=fetch_spec)
  File "/usr/lib/python2.7/dist-packages/bzrlib/decorators.py", line 218, in write_locked
    result = unbound(self, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/vf_repository.py", line 2584, in fetch
    find_ghosts=find_ghosts)
  File "/usr/lib/python2.7/dist-packages/bzrlib/fetch.py", line 77, in __init__
    self.__fetch()
  File "/usr/lib/python2.7/dist-packages/bzrlib/fetch.py", line 104, in __fetch
    self._fetch_everything_for_search(search_result)
  File "/usr/lib/python2.7/dist-packages/bzrlib/fetch.py", line 132, in _fetch_everything_for_search
    stream, from_format, [])
  File "/usr/lib/python2.7/dist-packages/bzrlib/remote.py", line 2881, in insert_stream
    (verb, path, resume_tokens) + lock_args, byte_stream)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/client.py", line 108, in call_with_body_stream
    expect_response_body=False)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/client.py", line 55, in _call_and_read_response
    return request.call_and_read_response()
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/client.py", line 157, in call_and_read_response
    return self._call(protocol_version)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/client.py", line 192, in _call
    expect_body=self.expect_response_body)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/message.py", line 308, in read_response_tuple
    _raise_smart_server_error(self.args)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/message.py", line 353, in _raise_smart_server_error
    raise errors.ErrorFromSmartServer(error_tuple)
ErrorFromSmartServer: Error received from smart server: ('error', "invalid literal for int() with base 10: 'None'")

On the server side I also get this error:
dom 2012-06-03 22:22:26 +0200
0.033 bazaar version: 2.3.4
0.033 bzr arguments: [u'serve', u'--inet', u'--directory=/home/bzr/repositories/bazitis-cascara@prometeus', u'--allow-writes']
0.047 looking for plugins in /home/bzr/.bazaar/plugins
0.083 looking for plugins in /usr/lib/python2.7/dist-packages/bzrlib/plugins
0.096 encoding stdout as osutils.get_user_encoding() 'UTF-8'
0.575 Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/request.py", line 356, in _call_converting_errors
    return callable(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/request.py", line 141, in execute
    return self.do(*args)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/branch.py", line 52, in do
    return self.do_with_branch(branch, *args)
  File "/usr/lib/python2.7/dist-packages/bzrlib/smart/branch.py", line 171, in do_with_branch
    revno, last_revision = branch.last_revision_info()
  File "/usr/lib/python2.7/dist-packages/bzrlib/decorators.py", line 138, in read_locked
    self.lock_read()
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 2502, in lock_read
    self.repository.lock_read()
  File "/usr/lib/python2.7/dist-packages/bzrlib/repofmt/pack_repo.py", line 2404, in lock_read
    self._refresh_data()
  File "/usr/lib/python2.7/dist-packages/bzrlib/repofmt/pack_repo.py", line 2328, in _refresh_data
    self._pack_collection.reload_pack_names()
  File "/usr/lib/python2.7/dist-packages/bzrlib/repofmt/pack_repo.py", line 2036, in reload_pack_names
    first_read = self.ensure_loaded()
  File "/usr/lib/python2.7/dist-packages/bzrlib/repofmt/pack_repo.py", line 1667, in ensure_loaded
    self._names[name] = self._parse_index_sizes(value)
  File "/usr/lib/python2.7/dist-packages/bzrlib/repofmt/pack_repo.py", line 1679, in _parse_index_sizes
    return tuple([int(digits) for digits in value.split(' ')])
ValueError: invalid literal for int() with base 10: 'None'

I found that changing this line on the bzrlib/repofmt/pack_repo.py file on the client and the server:
1100 return tuple([int(digits) for digits in value.split(' ')])

to this:
1100 return tuple([int(digits) for digits in value.replace('None', '0').split(' ')])

Solves the problem and the I can work with the repositories. But I'm not sure if this may introduce more errors or if it is safe.

Thanks

[Martin Packman (gz)]

It seems like you have a broken pack-names file in the repository in question, can you find it (inside .bzr/repository on the server) and attach it to the bug please?

If you want to see exactly where that dodgy value is, try running the following script:

    import pprint
    from bzrlib import (
        btree_index,
        transport,
    )

    url = "file:///path/to/repo/.bzr/repository"
    t = transport.get_transport(url)
    idx = btree_index.BTreeGraphIndex(t, "pack-names", None)
    pprint.pprint(list(idx.iter_all_entries()))

[David Caro (dcaro)]

Hi Martin,

thanks for the fast response.
The code that you gase me gives this output:
 [(<bzrlib.btree_index.BTreeGraphIndex object at 0x19437d0>,
  StaticTuple('17018ae1f7da6ec84cdd3e016c9325dd',),
  'None None None None None'),
 (<bzrlib.btree_index.BTreeGraphIndex object at 0x19437d0>,
  StaticTuple('f3b8ac41113eec69c1c8380f6d8b595f',),
  '166 165 264 72 181')]

(the pack-names file pf the server is attached).

do not hesitate to ask for anything else, this bug is blocking the development of the repo manager.

[John A Meinel (jameinel)]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 6/4/2012 1:00 AM, Martin Packman wrote:
> It seems like you have a broken pack-names file in the repository
> in question, can you find it (inside .bzr/repository on the server)
> and attach it to the bug please?
>
> If you want to see exactly where that dodgy value is, try running
> the following script:
>
> import pprint from bzrlib import ( btree_index, transport, )
>
> url = "file:///path/to/repo/.bzr/repository" t =
> transport.get_transport(url) idx = btree_index.BTreeGraphIndex(t,
> "pack-names", None) pprint.pprint(list(idx.iter_all_entries()))
>
> ** Changed in: bzr Status: New => Incomplete
>

Note that we also have:
  bzr dump-btree [--raw] .../.bzr/repository/pack-names

To debug these things as well without having to go into a python script.

John
=:->

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.9 (Cygwin)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAk/OHywACgkQJdeBCYSNAANjnQCgpkiEl9CKaVKhZ3k9cb//giDn
e1MAoKOaHcIcmyo0u1/zjLi+cxzfJwam
=iP4K
-----END PGP SIGNATURE-----

[Martin Packman (gz)]

Nice, another handy hidden command I've never used before. John, do we also have one for fixing/editing btrees? Or have you got any other suggestions for how to repair this branch?

David, do I understand correctly that this is the only one copy of the branch around? If you run `bzr check` on a copy after branching it with your patch, what is reported?

[John A Meinel (jameinel)]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 6/7/2012 12:50 AM, Martin Packman wrote:
> Nice, another handy hidden command I've never used before. John, do
> we also have one for fixing/editing btrees? Or have you got any
> other suggestions for how to repair this branch?
>
> David, do I understand correctly that this is the only one copy of
> the branch around? If you run `bzr check` on a copy after branching
> it with your patch, what is reported?
>

We don't have a simple script for editing it.

You can use something like your initial script. First, copy
'.bzr/repository/pack-names' somewhere else, so you can restore it if
something goes wrong.

import pprint
from bzrlib import (
    btree_index,
    transport,
)

url = "file:///path/to/repo/.bzr/repository"
t = transport.get_transport(url)
idx = btree_index.BTreeGraphIndex(t, "pack-names", None)
pprint.pprint(list(idx.iter_all_entries()))

out_idx = btree_index.BTreeBuilder()
for _, k, v in idx.iter_all_entries():
    if k == ?????:
        continue
    out_idx.add_node(k, v)

f, size = out_idx.finish()
del idx
t.put_bytes('pack-names', f.read())

In the '????' is any of the entries that you found to be None.

At that point, the repository should be updated.

John
=:->
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.9 (Cygwin)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAk/QjDIACgkQJdeBCYSNAAMAYgCgu4HrT7ouVnrAuLHXuQY2MWtO
yNIAoIvSlwJJmdD+dpREtfzPYargsYtH
=tr19
-----END PGP SIGNATURE-----

[David Caro (dcaro)]

  · @Martin:
The repository is an empty one, I can reproduce the failure just creating a new repo, branching it on a client, making some changes, and then trying to push it. Thankfully, there's no need to recover the data.

That only corrupts the repo on the server, on the client all the btree leafs have an integer size. I'm almost sure that the repo manager is the one breaking things up, but I don't know why :S

It's curious that the indexes files were created correctly (have non zero size), but the pack-names file has the None values.

In a few words, what the repo manager does is run a simple wrapper script around bzr serve and then uses a bzr plugin to check the user rights. To pass the user to the plugin the only way I've found is including it on the directory option of bzr serve (that's what the wrapper is for, at first, my intention was to check the user rights on the wrapper before calling bzr serve, but at that point in bzr you do not have this information, right?).

- ssh starts the wrapper with the username of the ssh key used
- the wrapper then starts bzr serve with the user on the directory option (this step is no more needed, you can start bzr serve from the ssh config)
- bzr server starts the plugin
- the plugin then registers a transport to check the user rights for each command executed

There is also a post_update hook, but it's not used yet at this point of the execution.

Running 'bzr check' throws the same exception:
Checking working tree at '/home/bzr/repositories/bazitis-admin'.
bzr: ERROR: exceptions.ValueError: invalid literal for int() with base 10: 'None'

Traceback (most recent call last):
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 924, in exception_to_return_code
    return the_callable(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 1124, in run_bzr
    ret = run(*run_argv)
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 689, in run_argv_aliases
    return self.run(**all_cmd_args)
  File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 711, in run
    return self._operation.run_simple(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 135, in run_simple
    self.cleanups, self.func, *args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 165, in _do_with_cleanups
    result = func(*args, **kwargs)
  File "/usr/lib/python2.7/dist-packages/bzrlib/builtins.py", line 3327, in run
    check_dwim(path, verbose, do_branch=branch, do_repo=repo, do_tree=tree)
  File "/usr/lib/python2.7/dist-packages/bzrlib/check.py", line 421, in check_dwim
    scan_tree(None, base_tree, needed_refs, to_unlock)
  File "/usr/lib/python2.7/dist-packages/bzrlib/check.py", line 394, in scan_tree
    tree.lock_read()
  File "/usr/lib/python2.7/dist-packages/bzrlib/workingtree_4.py", line 571, in lock_read
    self.branch.lock_read()
  File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 2502, in lock_read
    self.repository.lock_read()
  File "/usr/lib/python2.7/dist-packages/bzrlib/repofmt/pack_repo.py", line 2416, in lock_read
    self._refresh_data()
  File "/usr/lib/python2.7/dist...

[David Caro (dcaro)]

I'm gonna focus again on this bug, and I hope to solve it, because I'm eager to use bazaar inside my company, and if we do not have a similar way to handle users, we will have to change to git and gitosis/gitolite.

I'll keep you informed in case I solve the problem or in case I need more help.

Thanks again,
David

[David Caro (dcaro)]

I found this thread, that is giving me a lot of info :)

http://osdir.com/ml/bazaar/2010-02/msg00082.html

Thanks Andrew!

[David Caro (dcaro)]

I know what's failing, and I will reimplement most of the code to avoid this. I need to get a little more experience with bzrlib internals to get this working. I'm keepeing this bug open until the changes are made (one or two weeks at much).

Thanks!

[David Caro (dcaro)]

The post above was intended for this related bug: https://bugs.launchpad.net/bazitis/+bug/397452
Sorry

This bug can be closed, as the problem was that the custom transport implements was not returning the correct values for some of the reimplemented functions, and thus, not giving the correct sizes of the indexes when asked.

Now that I have more knowledge of the bzrlib internals I can redo most of the code using the right tools. I'm glad that this got solved :)

Thanks!

[Martin Packman (gz)]

That's great David. If you can work out a good place to insert a check in the bzrlib code that raises a better error that would be a good change too.

[David Caro (dcaro)]

I'll take a look into it, but first I want to have a working version of the
repo manager (hopefully this week).

David
El 30/06/2012 20:15, "Martin Packman" <email address hidden>
escribió:

> That's great David. If you can work out a good place to insert a check
> in the bzrlib code that raises a better error that would be a good
> change too.
>
> --
> You received this bug notification because you are subscribed to the bug
> report.
> https://bugs.launchpad.net/bugs/1008223
>
> Title:
> Error when pushing through smart server
>
> Status in Bazaar Version Control System:
> Invalid
>
> Bug description:
> I'm trying to develop a little repo manager like gitosis for git. And
> when doing the first push from the client to the server, this error
> shows up:
>
> bzr: ERROR: bzrlib.errors.ErrorFromSmartServer: Error received from
> smart server: ('error', "invalid literal for int() with base 10:
> 'None'")
>
> Traceback (most recent call last):
> File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 920,
> in exception_to_return_code
> return the_callable(*args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 1131,
> in run_bzr
> ret = run(*run_argv)
> File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 673,
> in run_argv_aliases
> return self.run(**all_cmd_args)
> File "/usr/lib/python2.7/dist-packages/bzrlib/commands.py", line 695,
> in run
> return self._operation.run_simple(*args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 136,
> in run_simple
> self.cleanups, self.func, *args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 166,
> in _do_with_cleanups
> result = func(*args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/builtins.py", line 1379,
> in run
> use_existing_dir=use_existing_dir, no_tree=no_tree)
> File "/usr/lib/python2.7/dist-packages/bzrlib/push.py", line 145, in
> _show_push_branch
> remember, create_prefix)
> File "/usr/lib/python2.7/dist-packages/bzrlib/controldir.py", line
> 425, in push_branch
> overwrite, stop_revision=revision_id)
> File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 1142,
> in push
> lossy, *args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3507,
> in push
> _override_hook_source_branch=_override_hook_source_branch)
> File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 132,
> in run
> self.cleanups, self.func, self, *args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/cleanup.py", line 166,
> in _do_with_cleanups
> result = func(*args, **kwargs)
> File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3562,
> in _push_with_bound_branches
> result = self._basic_push(overwrite, stop_revision)
> File "/usr/lib/python2.7/dist-packages/bzrlib/branch.py", line 3524,
> in _basic_push
> graph=graph)
> File "/usr/lib/python2.7/dist-packages/bzrlib/decorators.py", line
> 218, in write_locked
> result = unbound(self, *args, **kwargs)
> ...