diff --git a/CHANGES.txt b/CHANGES.txt index e2f3fd5..6184ce7 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -12,6 +12,10 @@ New Feature: Bugs fixed: +- Option incrementing (+=) and decrementing (-=) did not work well with + macro sections (<=). Fixes + https://bugs.launchpad.net/zc.buildout/+bug/435837 + - While checking for new versions of setuptools and buildout itself, compare requirement locations instead of requirement objects. diff --git a/src/zc/buildout/buildout.py b/src/zc/buildout/buildout.py index 44766f2..eed040b 100644 --- a/src/zc/buildout/buildout.py +++ b/src/zc/buildout/buildout.py @@ -160,13 +160,12 @@ class Buildout(UserDict.DictMixin): base = None - cloptions = dict( + override = dict( (section, dict((option, (value, 'COMMAND_LINE_VALUE')) for (_, option, value) in v)) for (section, v) in itertools.groupby(sorted(cloptions), lambda v: v[0]) ) - override = cloptions.get('buildout', {}).copy() # load user defaults, which override defaults if user_defaults: @@ -181,9 +180,6 @@ class Buildout(UserDict.DictMixin): _update(data, _open(os.path.dirname(config_file), config_file, [], data['buildout'].copy(), override)) - # apply command-line options - _update(data, cloptions) - self._annotated = copy.deepcopy(data) self._raw = _unannotate(data) self._data = {} @@ -1028,9 +1024,6 @@ class Options(UserDict.DictMixin): name = self.name __doing__ = 'Initializing section %s.', name - if '<' in self._raw: - self._raw = self._do_extend_raw(name, self._raw, []) - # force substitutions for k, v in self._raw.items(): if '${' in v: @@ -1051,33 +1044,6 @@ class Options(UserDict.DictMixin): self.recipe = recipe_class(buildout, name, self) buildout._parts.append(name) - def _do_extend_raw(self, name, data, doing): - if name == 'buildout': - return data - if name in doing: - raise zc.buildout.UserError("Infinite extending loop %r" % name) - doing.append(name) - try: - to_do = data.pop('<', None) - if to_do is None: - return data - __doing__ = 'Loading input sections for %r', name - - result = {} - for iname in to_do.split('\n'): - iname = iname.strip() - if not iname: - continue - raw = self.buildout._raw.get(iname) - if raw is None: - raise zc.buildout.UserError("No section named %r" % iname) - result.update(self._do_extend_raw(iname, raw, doing)) - - result.update(data) - return result - finally: - assert doing.pop() == name - def _dosub(self, option, v): __doing__ = 'Getting option %s:%s.', self.name, option seen = [(self.name, option)] @@ -1273,7 +1239,7 @@ def _open(base, filename, seen, dl_options, override): Recursively open other files based on buildout options found. """ - _update_section(dl_options, override) + _update_section(dl_options, override.get('buildout', {})) _dl_options = _unannotate_section(dl_options.copy()) is_temp = False download = zc.buildout.download.Download( @@ -1330,10 +1296,15 @@ def _open(base, filename, seen, dl_options, override): if extends: extends = extends.split() - eresult = _open(base, extends.pop(0), seen, dl_options, override) + eresult = _expand_macros(result, _open(base, extends.pop(0), seen, dl_options, override), override) + for fname in extends: - _update(eresult, _open(base, fname, seen, dl_options, override)) - result = _update(eresult, result) + _update(eresult, _expand_macros(eresult, _open(base, fname, seen, dl_options, override), override)) + + result = _update(eresult, _expand_macros(eresult, result, override)) + else: + # Macro expand the only section. + result = _expand_macros(result, result, override) if extended_by: self._logger.warn( @@ -1346,6 +1317,63 @@ def _open(base, filename, seen, dl_options, override): seen.pop() return result +def _expand_macros(expansion_base, config, overrides=None): + """Expands macros found in ``config`` by looking up the macro definitions + in ``expansion_base`` (which represents the overall buildout configuration + parsed so far). + + If ``overrides`` is given then both the ``expansion_base`` and ``config`` + are updated with it to override existing values before the expansion takes + place. + + Returns a new dictionary based on ``config`` with macros expanded. + """ + expanded = config.copy() + if overrides is not None: + for section in overrides: + expanded.setdefault(section, {}).update(overrides[section]) + # Note that we modify the expansion_place here and let the changes + # propagate. + expansion_base.setdefault(section, {}).update(overrides[section]) + + for section in config: + expanded[section] = _expand_section(section, expansion_base, expanded, []) + + return expanded + +def _expand_section(name, expansion_base, config, expanded): + """Expands macros in ``config[name]`` section using macro definitions from + ``expansion_base``. + """ + if name in expanded: + raise zc.buildout.UserError('Infinite section expansion loop %r' % name) + + macro_names, note = config[name].get('<', (None, None)) + expanded.append(name) + + if macro_names is None: + # No macros in use + return config[name] + + expanded_section = {} + for macro in macro_names.splitlines(): + macro = macro.strip() + if not macro: + continue + + # Expand recursively + if macro in config: + expanded_section.update(_expand_section(macro, expansion_base, config, expanded)) + elif macro in expansion_base: + expanded_section.update(_expand_section(macro, expansion_base, expansion_base, expanded)) + else: + raise zc.buildout.UserError('No section named %r available for section extension.' % macro) + + # Override any values we got from the macros with the section specific ones. + expanded_section.update(config[name]) + # Remove the macro option from the current section. + expanded_section.pop('<', None) + return expanded_section ignore_directories = '.svn', 'CVS' def _dir_hash(dir): @@ -1372,35 +1400,50 @@ def _dists_sig(dists): result.append(os.path.basename(location)) return result -def _update_section(s1, s2): - s2 = s2.copy() # avoid mutating the second argument, which is unexpected - for k, v in s2.items(): +def _expand_modifiers(source, section): + """Expands the +=/-= modifies used in ``section`` by looking up + in ``source`` in addition to ``section``. + + Returns a new dictionary containing the information from ``section`` + with modifiers expanded. + """ + expanded = {} + # We look up the reference sections for +/- operations in both the section + # itself and the additional source section. + lookup = source.copy() + lookup.update(section) + + for k, v in section.items(): v2, note2 = v if k.endswith('+'): key = k.rstrip(' +') - v1, note1 = s1.get(key, ("", "")) + v1, note1 = lookup.get(key, ("", "")) newnote = ' [+] '.join((note1, note2)).strip() - s2[key] = "\n".join((v1).split('\n') + - v2.split('\n')), newnote - del s2[k] + expanded[key] = "\n".join((v1).split('\n') + v2.split('\n')), newnote + lookup.update(expanded) elif k.endswith('-'): key = k.rstrip(' -') - v1, note1 = s1.get(key, ("", "")) + v1, note1 = lookup.get(key, ("", "")) newnote = ' [-] '.join((note1, note2)).strip() - s2[key] = ("\n".join( - [v for v in v1.split('\n') - if v not in v2.split('\n')]), newnote) - del s2[k] + expanded[key] = ("\n".join( + [v for v in v1.split('\n') if v not in v2.split('\n')]), newnote) + lookup.update(expanded) + else: + expanded[k] = v + + return expanded - s1.update(s2) +def _update_section(s1, s2): + s1.update(_expand_modifiers(s1, s2)) return s1 def _update(d1, d2): for section in d2: if section in d1: - d1[section] = _update_section(d1[section], d2[section]) + d1[section].update(_expand_modifiers(d1[section], d2[section])) else: - d1[section] = d2[section] + d1[section] = _expand_modifiers(d2[section], d2[section]) + return d1 def _recipe(options): diff --git a/src/zc/buildout/tests.py b/src/zc/buildout/tests.py index 94fc970..0b7b073 100644 --- a/src/zc/buildout/tests.py +++ b/src/zc/buildout/tests.py @@ -2653,6 +2653,155 @@ def increment_on_command_line(): recipe='zc.buildout:debug' """ +def bug_435837_macro_expansion_with_option_increments_and_extended_config(): + r""" + >>> write(sample_buildout, 'base.cfg', + ... ''' + ... [buildout] + ... parts = mypart + ... log-level = INFO + ... + ... [macro] + ... recipe = zc.buildout:debug + ... foo = + ... original_value_1 + ... original_value_2 + ... original_value_3 + ... + ... [mypart] + ... <= macro + ... ''') + + >>> write(sample_buildout, 'buildout.cfg', + ... ''' + ... [buildout] + ... extends = base.cfg + ... + ... [mypart] + ... foo += + ... extended_option_1 + ... extended_option_2 + ... foo -= + ... original_value_1 + ... ''') + + >>> print system(buildout), + Installing mypart. + foo='original_value_2\noriginal_value_3\n\nextended_option_1\nextended_option_2' + recipe='zc.buildout:debug' + """ + +def bug_435837_recursive_macro_expansion(): + r""" + >>> write(sample_buildout, 'buildout.cfg', + ... ''' + ... [buildout] + ... parts = + ... mypart1 + ... mypart2 + ... log-level = INFO + ... + ... [macro] + ... recipe = zc.buildout:debug + ... foo = foo_value + ... + ... [mypart1] + ... <= macro + ... bar = bar_value + ... + ... [mypart2] + ... <= mypart1 + ... foo = custom_value + ... ''') + + >>> print system(buildout) + Installing mypart1. + bar='bar_value' + foo='foo_value' + recipe='zc.buildout:debug' + Installing mypart2. + bar='bar_value' + foo='custom_value' + recipe='zc.buildout:debug' + + """ + +def bug_435837_recursive_macro_expansion__fails_with_dash_in_section_name(): + r""" + The use of dashes in the section named revealed an ordering related bug + when iterating over a section keys resulting in the following error + message. + + Error: missing option: project-one:recipe + + >>> write(sample_buildout, 'buildout.cfg', + ... ''' + ... [buildout] + ... parts = + ... project-one + ... project-two + ... log-level = INFO + ... + ... [project] + ... recipe = zc.buildout:debug + ... + ... [project-one] + ... <= project + ... + ... [project-two] + ... <= project-one + ... ''') + + >>> print system(buildout) + Installing project-one. + recipe='zc.buildout:debug' + Installing project-two. + recipe='zc.buildout:debug' + + """ + +def bug_435837_recursive_macro_expansion_with_multiple_base_macros(self): + r""" + >>> write(sample_buildout, 'buildout.cfg', + ... ''' + ... [buildout] + ... parts = + ... project-one + ... project-two + ... log-level = INFO + ... + ... [macro1] + ... recipe = zc.buildout:debug + ... foo = one + ... + ... [macro2] + ... recipe = zc.buildout:debug + ... foo = two + ... bar = three + ... + ... [project-one] + ... <= + ... macro1 + ... macro2 + ... + ... [project-two] + ... <= + ... macro2 + ... macro1 + ... ''') + + >>> print system(buildout) + Installing project-one. + bar='three' + foo='two' + recipe='zc.buildout:debug' + Installing project-two. + bar='three' + foo='one' + recipe='zc.buildout:debug' + + """ + ###################################################################### def create_sample_eggs(test, executable=sys.executable):