Xtrabackup Doesn't free File Handlers while running prepare of a compacted backup

[Daniel Chen]

Attaching some interesting observation during the xtrabackup --prepare run with rebuild-indexes:

[root@backend-db-drp lib]# df -kh
Filesystem Size Used Avail Use% Mounted on
/dev/mapper/vg_sys-LogVol00
                       49G 1.4G 45G 3% /
tmpfs 253G 0 253G 0% /dev/shm
/dev/md0 485M 33M 428M 7% /boot
/dev/sdc 7.9T 6.5T 1.1T 87% /var/lib/mysql

[root@backend-db-drp lib]# du -sh mysql
4.5T mysql
[root@backend-db-drp lib]#

I've asked myself where did 2 TB go? (looks like xtrabackup doesn't free deleted file handlers)

[root@backend-db-drp lib]# lsof +L1|less

COMMAND PID USER FD TYPE DEVICE SIZE/OFF NLINK NODE NAME
xtrabacku 17224 root 4r REG 8,32 637755392 0 240779835 /var/lib/mysql/trc_backend/namex_raw_data#P#p20130717.ibd.tmp (deleted)
xtrabacku 17224 root 5r REG 8,32 283758592 0 240780015 /var/lib/mysql/trc_backend/namex_raw_data#P#p20130615.ibd.tmp (deleted)
xtrabacku 17224 root 6r REG 8,32 1127809024 0 240780084 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130412.ibd.tmp (deleted)
xtrabacku 17224 root 7r REG 8,32 3150970880 0 240779350 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130614.ibd.tmp (deleted)
xtrabacku 17224 root 8r REG 8,32 197529600 0 240779418 /var/lib/mysql/trc_backend/monitor_events#P#p20130724.ibd.tmp (deleted)
xtrabacku 17224 root 9r REG 8,32 163033088 0 240779353 /var/lib/mysql/trc_backend/monitor_events#P#p20130705.ibd.tmp (deleted)
xtrabacku 17224 root 10r REG 8,32 439054336 0 240779983 /var/lib/mysql/trc_backend/namex_raw_data#P#p20130629.ibd.tmp (deleted)
xtrabacku 17224 root 11r REG 8,32 436830208 0 240779942 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130608.ibd.tmp (deleted)
xtrabacku 17224 root 12r REG 8,32 183869440 0 240779623 /var/lib/mysql/trc_backend/monitor_events#P#p20130710.ibd.tmp (deleted)
xtrabacku 17224 root 13r REG 8,32 169086976 0 240779831 /var/lib/mysql/trc_backend/monitor_events#P#p20130614.ibd.tmp (deleted)
xtrabacku 17224 root 14r REG 8,32 251850752 0 240779330 /var/lib/mysql/trc_backend/monitor_events#P#p20130804.ibd.tmp (deleted)
xtrabacku 17224 root 15r REG 8,32 508628992 0 240779451 /var/lib/mysql/trc_backend/namex_raw_data#P#p20130614.ibd.tmp (deleted)
xtrabacku 17224 root 16r REG 8,32 24084480 0 240780105 /var/lib/mysql/trc_backend/trend_to_namex.ibd (deleted)
xtrabacku 17224 root 17r REG 8,32 147456 0 240779760 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130628.ibd.tmp (deleted)
xtrabacku 17224 root 18r REG 8,32 864681984 0 240779909 /var/lib/mysql/trc_backend/component_calibration_v2#P#p20130707.ibd.tmp (deleted)
xtrabacku 17224 root 19r REG 8,32 131072 0 240779978 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130702.ibd.tmp (deleted)
xtrabacku 17224 root 20r REG 8,32 24608768 0 240779299 /var/lib/mysql/trc_backend/trend_score.ibd (deleted)
xtrabacku 17224 root 21r REG 8,32 1151287296 0 240779828 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130421.ibd.tmp (deleted)
xtrabacku 17224 root 22r REG 8,32 131072 0 240780039 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130716.ibd.tmp (deleted)
xtrabacku 17224 root 23r REG 8,32 494501888 0 240779319 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130210.ibd.tmp (deleted)
xtrabacku 17224 root 24r REG 8,32 147456 0 240780116 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130706.ibd.tmp (deleted)
xtrabacku 17224 root 25r REG 8,32 2464694272 0 240779988 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130615.ibd.tmp (deleted)
xtrabacku 17224 root 26r REG 8,32 1690959872 0 240780253 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130509.ibd.tmp (deleted)
xtrabacku 17224 root 27r REG 8,32 165842944 0 240780122 /var/lib/mysql/trc_backend/monitor_events#P#p20130616.ibd.tmp (deleted)
xtrabacku 17224 root 28r REG 8,32 1211695104 0 240780046 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130523.ibd.tmp (deleted)
xtrabacku 17224 root 29r REG 8,32 131072 0 240780159 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130817.ibd.tmp (deleted)
xtrabacku 17224 root 30r REG 8,32 939393024 0 240780204 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130408.ibd.tmp (deleted)
xtrabacku 17224 root 31r REG 8,32 131072 0 240779497 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130811.ibd.tmp (deleted)
xtrabacku 17224 root 32r REG 8,32 1048576 0 240779619 /var/lib/mysql/trc_backend/campaign_item_monitoring_log.ibd.tmp (deleted)
xtrabacku 17224 root 33r REG 8,32 619298816 0 240780138 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130402.ibd.tmp (deleted)
xtrabacku 17224 root 34r REG 8,32 1155612672 0 240779653 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130415.ibd.tmp (deleted)
xtrabacku 17224 root 35r REG 8,32 2063138816 0 240779876 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v2#P#p20130720.ibd.tmp (deleted)
xtrabacku 17224 root 36r REG 8,32 131072 0 240779686 /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130720.ibd.tmp (deleted)
xtrabacku 17224 root 37r REG 8,32 1313947648 0 240779617 /var/lib/mysql/trc_backend/component_calibration_v2#P#p20130723.ibd.tmp (deleted)
xtrabacku 17224 root 38r REG 8,32 524943360 0 240779756 /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130306.ibd.tmp (deleted)
xtrabacku 17224 root 39r REG 8,32 694968320 0 240779505 /var/lib/mysql/trc_backend/namex_raw_data#P#p20130802.ibd.tmp (deleted)
xtrabacku 17224 root 40r REG 8,32 576294912 0 240780158 /var/lib/mysql/trc_backend/namex_raw_data#P#p20130701.ibd.tmp (deleted)
xtrabacku 17224 root 41r REG 8,32 1008107520 0 240779483 /var/lib/mysql/trc_backend/component_calibration_v2#P#p20130708.ibd.tmp (deleted)

[Alexey Kopytov (akopytov)]

Thanks. The same problem has been reported as bug #1111380, but I missed one important detail in my code analysis back then. I now see where this problem comes from. Re-opening bug #1111380 and closing this one as a duplicate.

[Daniel Chen (chendaniel3)]

Thank you Alexey. I appreciate your fast response and the will to fix it
soon.
On Aug 21, 2013 8:00 AM, "Alexey Kopytov" <email address hidden>
wrote:

> *** This bug is a duplicate of bug 1111380 ***
> https://bugs.launchpad.net/bugs/1111380
>
> Thanks. The same problem has been reported as bug #1111380, but I missed
> one important detail in my code analysis back then. I now see where this
> problem comes from. Re-opening bug #1111380 and closing this one as a
> duplicate.
>
> ** This bug has been marked a duplicate of bug 1111380
> --rebuild-indexes holds on to disk space during --prepare
>
> --
> You received this bug notification because you are subscribed to the bug
> report.
> https://bugs.launchpad.net/bugs/1214537
>
> Title:
> Xtrabackup Doesn't free File Handlers while running prepare of a
> compacted backup
>
> Status in Percona XtraBackup:
> New
>
> Bug description:
> Attaching some interesting observation during the xtrabackup --prepare
> run with rebuild-indexes:
>
> [root@backend-db-drp lib]# df -kh
> Filesystem Size Used Avail Use% Mounted on
> /dev/mapper/vg_sys-LogVol00
> 49G 1.4G 45G 3% /
> tmpfs 253G 0 253G 0% /dev/shm
> /dev/md0 485M 33M 428M 7% /boot
> /dev/sdc 7.9T 6.5T 1.1T 87% /var/lib/mysql
>
> [root@backend-db-drp lib]# du -sh mysql
> 4.5T mysql
> [root@backend-db-drp lib]#
>
> I've asked myself where did 2 TB go? (looks like xtrabackup doesn't
> free deleted file handlers)
>
> [root@backend-db-drp lib]# lsof +L1|less
>
> COMMAND PID USER FD TYPE DEVICE SIZE/OFF NLINK NODE NAME
> xtrabacku 17224 root 4r REG 8,32 637755392 0 240779835
> /var/lib/mysql/trc_backend/namex_raw_data#P#p20130717.ibd.tmp (deleted)
> xtrabacku 17224 root 5r REG 8,32 283758592 0 240780015
> /var/lib/mysql/trc_backend/namex_raw_data#P#p20130615.ibd.tmp (deleted)
> xtrabacku 17224 root 6r REG 8,32 1127809024 0 240780084
> /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130412.ibd.tmp
> (deleted)
> xtrabacku 17224 root 7r REG 8,32 3150970880 0 240779350
> /var/lib/mysql/trc_backend/component_calibration_v1#P#p20130614.ibd.tmp
> (deleted)
> xtrabacku 17224 root 8r REG 8,32 197529600 0 240779418
> /var/lib/mysql/trc_backend/monitor_events#P#p20130724.ibd.tmp (deleted)
> xtrabacku 17224 root 9r REG 8,32 163033088 0 240779353
> /var/lib/mysql/trc_backend/monitor_events#P#p20130705.ibd.tmp (deleted)
> xtrabacku 17224 root 10r REG 8,32 439054336 0 240779983
> /var/lib/mysql/trc_backend/namex_raw_data#P#p20130629.ibd.tmp (deleted)
> xtrabacku 17224 root 11r REG 8,32 436830208 0 240779942
> /var/lib/mysql/trc_backend/pair_algorithm_evaluations_v1#P#p20130608.ibd.tmp
> (deleted)
> xtrabacku 17224 root 12r REG 8,32 183869440 0 240779623
> /var/lib/mysql/trc_backend/monitor_events#P#p20130710.ibd.tmp (deleted)
> xtrabacku 17224 root 13r REG 8,32 169086976 0 240779831
> /var/lib/mysql/trc_backend/monitor_events#P#p20130614.ibd.tmp (delet...