Comment 3 for bug 1810048

I just hit the same issue. Here are some notes I was dumping to when looking over the issue:

It looks like horizon is bombing when trying to set up project panels under /usr/share/openstack-dashboard as it doesn't exist. It look like the horizon lib dir was modified to support package installs. Recent python3 changes modify the horizon_lib_dir path without modifying the horizon_dashboard_panel_dir to match on source based installs.

# Example error:

TASK [os_horizon : Enable project panels] *************************************************************************************************************************************************************************
failed: [infra1_horizon_container-63cf055a] (item={'uid': 0, 'woth': False, 'mtime': 1571255171.4404233, 'inode': 1066421, 'isgid': False, 'size': 1070, 'roth': True, 'isuid': False, 'isreg': True, 'pw_name': 'r
oot', 'gid': 0, 'ischr': False, 'wusr': True, 'xoth': False, 'rusr': True, 'nlink': 1, 'issock': False, 'rgrp': True, 'gr_name': 'root', 'path': '/openstack/venvs/horizon-20.0.0.0b2.dev27/lib/python3.6/dist-pack
ages/heat_dashboard/enabled/_1640_project_template_versions_panel.py', 'xusr': False, 'atime': 1571255210.9088569, 'isdir': False, 'ctime': 1571255211.092859, 'isblk': False, 'xgrp': False, 'dev': 64772, 'wgrp':
 False, 'isfifo': False, 'mode': '0644', 'islnk': False}) => {"ansible_loop_var": "item", "changed": false, "item": {"atime": 1571255210.9088569, "ctime": 1571255211.092859, "dev": 64772, "gid": 0, "gr_name": "r
oot", "inode": 1066421, "isblk": false, "ischr": false, "isdir": false, "isfifo": false, "isgid": false, "islnk": false, "isreg": true, "issock": false, "isuid": false, "mode": "0644", "mtime": 1571255171.440423
3, "nlink": 1, "path": "/openstack/venvs/horizon-20.0.0.0b2.dev27/lib/python3.6/dist-packages/heat_dashboard/enabled/_1640_project_template_versions_panel.py", "pw_name": "root", "rgrp": true, "roth": true, "rus
r": true, "size": 1070, "uid": 0, "wgrp": false, "woth": false, "wusr": true, "xgrp": false, "xoth": false, "xusr": false}, "msg": "Error while linking: [Errno 2] No such file or directory", "path": "/usr/share/
openstack-dashboard/openstack_dashboard/local/enabled/_1640_project_template_versions_panel.py"}

# Related commit adding package support: https://github.com/openstack/openstack-ansible-os_horizon/commit/2af1b7a6c4644e05b41fcdad5d546dc4662560ff
# Related python3 commit breaking panel tasks: https://github.com/openstack/openstack-ansible-os_horizon/blame/3018868844c6d78a2120f0fba83b41596e6b2dfb/tasks/horizon_install_source.yml#L81

# Found in /etc/ansible/roles/os_horizon/defaults/main.yml

horizon_lib_dir: "{{ _horizon_lib_dir }}"

# Found in /etc/ansible/roles/os_horizon/vars/debian.yml

_horizon_lib_dir: "/usr/share/openstack-dashboard"
horizon_dashboard_panel_dir: "{{ _horizon_lib_dir }}/openstack_dashboard/local/enabled"

# Found in /etc/ansible/roles/os_horizon/tasks/horizon_install_source.yml

    !!!! (This overrides what is set in the defaults, but leaves horizon_dashboard_panel_dir pointing to the old directory.)

- name: Set python lib dir fact
  set_fact:
    horizon_lib_dir: "{{ _horizon_python_venv_details.files[0].path }}/dist-packages"

    !!!! (This links /openstack/venvs/horizon-20.0.0.0b2.dev27/lib/python3.6/disk-packages /openstack/venvs/horizon-20.0.0.0b2.dev27/lib/python3.6/site-packages) ?

- name: Create horizon link for venv
  file:
    src: "{{ horizon_lib_dir | dirname }}/site-packages"
    dest: "{{ horizon_lib_dir }}"
    owner: "{{ horizon_system_user_name }}"
    group: "{{ horizon_system_group_name }}"
    state: "link"

    !!!! (This is creating directories under /openstack/venvs/horizon-20.0.0.0b2.dev27/lib/python3.6/dist-packages/)

- name: Create static horizon dir
  file:
    path: "{{ item.path }}"
    state: "directory"
    owner: "{{ item.owner|default(horizon_system_user_name) }}"
    group: "{{ item.group|default(horizon_system_group_name) }}"
  with_items:
    - { path: "{{ horizon_lib_dir }}/static", mode: "2755" }
    - { path: "{{ horizon_lib_dir }}/openstack_dashboard", mode: "2755" }
    - { path: "{{ horizon_lib_dir }}/openstack_dashboard/local", mode: "2755" }
    - { path: "{{ horizon_lib_dir }}/openstack_dashboard/local/enabled", mode: "2755" }

    !!!! (This registers any dashboards under /openstack/venvs/horizon-20.0.0.0b2.dev27/lib/python3.6/dist-packages/ with dashboard or ui in the name)

- name: Registering dashboards
  find:
    paths: "{{ horizon_lib_dir }}"
    patterns: "^.*(dashboard|ui)$"
    file_type: directory
    use_regex: yes
    excludes: "openstack_dashboard"
    recurse: no
  register: found_dashboards

    !!!! (This goes through files in found_dashboards and registers any that are named for panels)
- name: Registering panels
  find:
    paths: |-
      {% set dashboard_path = [] %}
      {% for dashboard in found_dashboards.files %}
      {% for path in _dashboard_panels_location %}
      {% set _ = dashboard_path.append(dashboard.path + path) %}
      {% endfor %}
      {% endfor %}
      {{ dashboard_path }}
    patterns: ["^_[0-9]{2,4}_.*.py$"]
    file_type: file
    use_regex: yes
  register: found_panels

    !!!! (This is trying to link the found_panels to /usr/share/openstack-dashboard which doesn't exist!!!. Looks like another set_fact is needed for horizon_dashboard_panel_dir after horizon_lib_dir is overridden?)

- name: Enable project panels
  file:
    src: "{{ item.path }}"
    path: "{{ horizon_dashboard_panel_dir }}/{{ item.path|basename }}"
    state: link
  with_items: "{{ found_panels.files }}"
  notify:
    - Compile messages
    - Restart apache2