[CentOS] compare zfs xfs and jfs o

Sun Aug 12 23:47:31 UTC 2012
Dennis Clarke <dclarke at blastwave.org>

> 
> Whatever you do, don't use bemchmarks to compare Linux with other OS, 
> Linux cheats.
> 
> A benchmark tries to do something completed and then get the time for 
> that but 
> I've seen Linux to try it's best to prevent this completed set of 
> actions to 
> happen withing a known time.
> 
> If I unpack the linux kernel sources on Solaris and UFS using star 
> (that by 
> default calls fsync() for each file) everything is stable on disk when 
> star 
> finishes.
> 

A little data never hurts. Even if the numbers mean little.

test 1 - Debian Linux 6.0.5 on x86_64 

root at tfs01:/usr/local# uname -a 
Linux tfs01 2.6.32-5-amd64 #1 SMP Sun May 6 04:00:17 UTC 2012 x86_64 GNU/Linux
root at tfs01:/usr/local# cat /etc/debian_version 
6.0.5

root at tfs01:/usr/local# cd src
root at tfs01:/usr/local/src# ls -a 
.  ..
root at tfs01:/usr/local/src# 
root at tfs01:/usr/local/src# wget http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.5.1.tar.bz2
--2012-08-12 20:39:18--  http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.5.1.tar.bz2
Resolving www.kernel.org... 149.20.4.69, 149.20.20.133
Connecting to www.kernel.org|149.20.4.69|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 80981090 (77M) [application/x-bzip2]
Saving to: `linux-3.5.1.tar.bz2'

100%[======================================>] 80,981,090   871K/s   in 92s     

2012-08-12 20:40:50 (861 KB/s) - `linux-3.5.1.tar.bz2' saved [80981090/80981090]

root at tfs01:/usr/local/src# 

root at tfs01:/usr/local/src# time -p openssl dgst -sha256 linux-3.5.1.tar.bz2
SHA256(linux-3.5.1.tar.bz2)= 78f8553c7cbc09d9c12d45c7f6ec82f964bf35d324832bede273ba2f6fe3e3ed
real 0.85
user 0.78
sys 0.02

CPU type : 

root at tfs01:/usr/local/src# grep -E "^processor|^vendor_id|^model\ name" /proc/cpuinfo 
processor       : 0
vendor_id       : AuthenticAMD
model name      : AMD Opteron(TM) Processor 6272                 
processor       : 1
vendor_id       : AuthenticAMD
model name      : AMD Opteron(TM) Processor 6272                 
processor       : 2
vendor_id       : AuthenticAMD
model name      : AMD Opteron(TM) Processor 6272                 
processor       : 3
vendor_id       : AuthenticAMD
model name      : AMD Opteron(TM) Processor 6272    

root at tfs01:/usr/local/src# mkdir foo
root at tfs01:/usr/local/src# cd foo
root at tfs01:/usr/local/src/foo# /opt/schily/bin/star -x -bz -xdir -xdot -U -fs=64m -fifostats -time file=../linux-3.5.1.tar.bz2 ; find . -type f | wc -l 
star: fifo had 46849 puts 80626 gets.
star: fifo was 1 times empty and 12 times full.
star: fifo held 67112960 bytes max, size was 67112960 bytes
star: fifo had 5 moves, total of 25600 moved bytes
star: fifo is 0% full (2k), size 65540k.
star: 46849 blocks + 0 bytes (total of 479733760 bytes = 468490.00k).
star: Total time 73.528sec (6371 kBytes/sec)
39096

root at tfs01:/usr/local/src/foo# cd ..  
root at tfs01:/usr/local/src# time -p rm -rf foo 
real 0.97
user 0.01
sys 0.94
root at tfs01:/usr/local/src# mkdir doo
root at tfs01:/usr/local/src# cd doo
root at tfs01:/usr/local/src/doo# which tar 
/bin/tar

root at tfs01:/usr/local/src/doo# time -p tar --use-compress-program /bin/bzip2 -xf ../linux-3.5.1.tar.bz2 ; find . -type f | wc -l 
real 25.14
user 24.03
sys 4.48
39096

Let's try that without compression involved.

root at tfs01:/usr/local/src# bunzip2 linux-3.5.1.tar.bz2 
root at tfs01:/usr/local/src# 
root at tfs01:/usr/local/src# ls -lApb 
total 468956
-rw-r--r-- 1 root staff 479733760 Aug  9 15:44 linux-3.5.1.tar
root at tfs01:/usr/local/src# mkdir foo
root at tfs01:/usr/local/src# cd foo
root at tfs01:/usr/local/src/foo# time -p /opt/schily/bin/star -x -xdir -xdot -U -fs=64m -fifostats -time file=../linux-3.5.1.tar                  
star: fifo had 46849 puts 80626 gets.
star: fifo was 1 times empty and 19 times full.
star: fifo held 67112960 bytes max, size was 67112960 bytes
star: fifo had 5 moves, total of 25600 moved bytes
star: fifo is 0% full (2k), size 65540k.
star: 46849 blocks + 0 bytes (total of 479733760 bytes = 468490.00k).
star: Total time 73.252sec (6395 kBytes/sec)
real 73.33
user 0.53
sys 5.77

root at tfs01:/usr/local/src/foo# cd ..
root at tfs01:/usr/local/src# rm -rf foo
root at tfs01:/usr/local/src# mkdir doo
root at tfs01:/usr/local/src# cd doo
root at tfs01:/usr/local/src/doo# time -p tar -xf ../linux-3.5.1.tar    
real 3.13
user 0.22
sys 2.80

3 seconds ? I find that a tad hard to believe. 

test 2 - Solaris 10 on HP Server x86_64 with ZFS 

$ uname -a 
SunOS foxtrot 5.10 Generic_147441-19 i86pc i386 i86pc
$ cat /etc/release 
                    Oracle Solaris 10 8/11 s10x_u10wos_17b X86
  Copyright (c) 1983, 2011, Oracle and/or its affiliates. All rights reserved.
                            Assembled 23 August 2011

$ zpool list
NAME            SIZE  ALLOC   FREE    CAP  HEALTH  ALTROOT
foxtrot_rpool  1.62T  66.1G  1.56T     3%  ONLINE  -

$ zpool get all foxtrot_rpool
NAME           PROPERTY       VALUE                               SOURCE
foxtrot_rpool  size           1.62T                               -
foxtrot_rpool  capacity       3%                                  -
foxtrot_rpool  altroot        -                                   default
foxtrot_rpool  health         ONLINE                              -
foxtrot_rpool  guid           14156136675261437976                default
foxtrot_rpool  version        29                                  default
foxtrot_rpool  bootfs         foxtrot_rpool/ROOT/s10x_u10wos_17b  local
foxtrot_rpool  delegation     on                                  default
foxtrot_rpool  autoreplace    off                                 default
foxtrot_rpool  cachefile      -                                   default
foxtrot_rpool  failmode       continue                            local
foxtrot_rpool  listsnapshots  on                                  default
foxtrot_rpool  autoexpand     off                                 default
foxtrot_rpool  free           1.56T                               -
foxtrot_rpool  allocated      66.1G                               -
foxtrot_rpool  readonly       off                                 -

$ psrinfo -pv 
The physical processor has 12 virtual processors (0 2 4 6 8 10 12 14 16 18 20 22)
  x86 (chipid 0x0 GenuineIntel family 6 model 44 step 2 clock 3467 MHz)
        Intel(r) Xeon(r) CPU           X5690  @ 3.47GHz
The physical processor has 12 virtual processors (1 3 5 7 9 11 13 15 17 19 21 23)
  x86 (chipid 0x1 GenuineIntel family 6 model 44 step 2 clock 3467 MHz)
        Intel(r) Xeon(r) CPU           X5690  @ 3.47GHz

$ /usr/sfw/bin/wget http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.5.1.tar.bz2
--2012-08-12 17:09:38--  http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.5.1.tar.bz2
Resolving www.kernel.org... 149.20.20.133, 149.20.4.69
Connecting to www.kernel.org|149.20.20.133|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 80981090 (77M) [application/x-bzip2]
Saving to: `linux-3.5.1.tar.bz2'

100%[======================================>] 80,981,090   605K/s   in 2m 13s  

2012-08-12 17:11:51 (596 KB/s) - `linux-3.5.1.tar.bz2' saved [80981090/80981090]

$ ptime digest -a sha256 linux-3.5.1.tar.bz2
78f8553c7cbc09d9c12d45c7f6ec82f964bf35d324832bede273ba2f6fe3e3ed

real        0.552
user        0.496
sys         0.039

Strangely the openssl in Solaris 10 is SHA256 ignorant. 

However one may use the mdigest in schilytools also : 

$ ptime /opt/schily/bin/mdigest -a sha256 linux-3.5.1.tar.bz2
78f8553c7cbc09d9c12d45c7f6ec82f964bf35d324832bede273ba2f6fe3e3ed  linux-3.5.1.tar.bz2

real        0.576
user        0.562
sys         0.014

Regardless .. let's do a test here :

$ mkdir foo
$ cd foo
$ ptime /opt/schily/bin/star -x -bz -xdir -xdot -U -fs=64m -fifostats -time file=../linux-3.5.1.tar.bz2
star: fifo had 46849 puts 80641 gets.
star: fifo was 14 times empty and 0 times full.
star: fifo held 22493696 bytes max, size was 67112960 bytes
star: fifo had 5 moves, total of 25600 moved bytes
star: fifo is 0% full (2k), size 65540k.
star: 46849 blocks + 0 bytes (total of 479733760 bytes = 468490.00k).
star: Total time 16.234sec (28858 kBytes/sec)

real       16.238
user       15.890
sys         3.738

There is also a GNU tar somewhere in Solaris 10 : 

$ /usr/sfw/bin/gtar --version 
tar (GNU tar) 1.26
Copyright (C) 2011 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>.
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.

Written by John Gilmore and Jay Fenlason.
$ which bzip2 
/usr/bin/bzip2

$ cd ..
$ rm -rf foo 
$ mkdir doo
$ cd doo

$ ptime /usr/sfw/bin/gtar --use-compress-program /usr/bin/bzip2 -xf ../linux-3.5.1.tar.bz2

real       16.597
user       16.484
sys         2.500
$ 

Also if we avoid the compression : 

$ cd ..
$ rm -rf doo 
$ cd ..
$ rm -rf doo 
$ 
$ ls 
linux-3.5.1.tar.bz2
$ bunzip2 linux-3.5.1.tar.bz2 
$ mkdir foo
$ cd foo
$ ptime /opt/schily/bin/star -x -xdir -xdot -U -fs=64m -fifostats -time file=../linux-3.5.1.tar
star: fifo had 46849 puts 80626 gets.
star: fifo was 1 times empty and 18 times full.
star: fifo held 67112448 bytes max, size was 67112960 bytes
star: fifo had 5 moves, total of 25600 moved bytes
star: fifo is 0% full (2k), size 65540k.
star: 46849 blocks + 0 bytes (total of 479733760 bytes = 468490.00k).
star: Total time 7.099sec (65993 kBytes/sec)

real        7.103
user        0.314
sys         3.376

$ cd ..
$ rm -rf foo 
$ mkdir doo
$ cd doo
$ ptime /usr/sfw/bin/gtar -xf ../linux-3.5.1.tar

real        2.579
user        0.280
sys         2.165
$ cd ..
$ rm -rf doo

TEST 3 - Solaris 10 on UltraSparc III and UFS filesystem 

jupiter-sparc-SunOS5.10 # uname -a 
SunOS jupiter 5.10 Generic_147440-20 sun4u sparc SUNW,Sun-Fire-480R

jupiter-sparc-SunOS5.10 # cat /etc/release 
                       Solaris 10 5/08 s10s_u5wos_10 SPARC
           Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
                        Use is subject to license terms.
                             Assembled 24 March 2008

jupiter-sparc-SunOS5.10 # psrinfo -pv 
The physical processor has 1 virtual processor (0)
  UltraSPARC-III+ (portid 0 impl 0x15 ver 0x23 clock 900 MHz)
The physical processor has 1 virtual processor (1)
  UltraSPARC-III+ (portid 1 impl 0x15 ver 0x23 clock 900 MHz)
The physical processor has 1 virtual processor (2)
  UltraSPARC-III+ (portid 2 impl 0x15 ver 0x23 clock 900 MHz)
The physical processor has 1 virtual processor (3)
  UltraSPARC-III+ (portid 3 impl 0x15 ver 0x23 clock 900 MHz)

jupiter-sparc-SunOS5.10 # df -F ufs -h
Filesystem             Size   Used  Available Capacity  Mounted on
/dev/md/dsk/d0          16G   9.4G       6.2G    61%    /
/dev/md/dsk/d3         7.8G   4.9G       2.6G    66%    /var

jupiter-sparc-SunOS5.10 # pwd
/usr/local/src
jupiter-sparc-SunOS5.10 # ls -a 
.   ..

jupiter-sparc-SunOS5.10 # /usr/sfw/bin/wget http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.5.1.tar.bz2
--2012-08-12 22:03:01--  http://www.kernel.org/pub/linux/kernel/v3.0/linux-3.5.1.tar.bz2
Resolving www.kernel.org... 149.20.20.133, 149.20.4.69
Connecting to www.kernel.org|149.20.20.133|:80... connected.
HTTP request sent, awaiting response... 200 OK
Length: 80981090 (77M) [application/x-bzip2]
Saving to: `linux-3.5.1.tar.bz2'

     0K ........ ........ ........ ........ ........ ........  3%  461K 2m45s
  3072K ........ ........ ........ ........ ........ ........  7%  476K 2m36s
 73728K ........ ........ ........ ........ ........ ........ 97%  510K 5s
 76800K ........ ........ ........ ........ ...              100%  512K=2m36s

2012-08-12 22:05:38 (505 KB/s) - `linux-3.5.1.tar.bz2' saved [80981090/80981090]

jupiter-sparc-SunOS5.10 # pwd
/usr/local/src

jupiter-sparc-SunOS5.10 # ptime digest -a sha256 linux-3.5.1.tar.bz2
78f8553c7cbc09d9c12d45c7f6ec82f964bf35d324832bede273ba2f6fe3e3ed

real        3.699
user        3.286
sys         0.264
jupiter-sparc-SunOS5.10 # ptime /opt/schily/bin/mdigest -a sha256 linux-3.5.1.tar.bz2
78f8553c7cbc09d9c12d45c7f6ec82f964bf35d324832bede273ba2f6fe3e3ed  linux-3.5.1.tar.bz2

real        3.136
user        2.964
sys         0.125
jupiter-sparc-SunOS5.10 # mkdir foo
jupiter-sparc-SunOS5.10 # cd foo
jupiter-sparc-SunOS5.10 # pwd
/usr/local/src/foo
jupiter-sparc-SunOS5.10 # ptime /opt/schily/bin/star -x -bz -xdir -xdot -U -fs=64m -fifostats -time file=../linux-3.5.1.tar.bz2
star: fifo had 46849 puts 80626 gets.
star: fifo was 1 times empty and 17 times full.
star: fifo held 67109888 bytes max, size was 67112960 bytes
star: fifo had 5 moves, total of 25600 moved bytes
star: fifo is 0% full (2k), size 65540k.
star: 46849 blocks + 0 bytes (total of 479733760 bytes = 468490.00k).
star: Total time 955.454sec (490 kBytes/sec)

real    15:55.612
user     1:01.947
sys        31.922

jupiter-sparc-SunOS5.10 # cd ..
jupiter-sparc-SunOS5.10 # ptime rm -rf foo

real     2:14.021
user        0.501
sys         7.249

jupiter-sparc-SunOS5.10 # mkdir doo
jupiter-sparc-SunOS5.10 # cd doo
jupiter-sparc-SunOS5.10 # ptime /usr/sfw/bin/gtar --use-compress-program /usr/bin/bzip2 -xf ../linux-3.5.1.tar.bz2

real     4:40.456
user     1:04.095
sys        19.713

jupiter-sparc-SunOS5.10 # cd ..
jupiter-sparc-SunOS5.10 # ptime rm -rf doo

real     2:14.808
user        0.502
sys         7.235

jupiter-sparc-SunOS5.10 # ptime bunzip2 linux-3.5.1.tar.bz2

real     1:05.220
user     1:00.120
sys         3.073

jupiter-sparc-SunOS5.10 # mkdir foo
jupiter-sparc-SunOS5.10 # cd foo
jupiter-sparc-SunOS5.10 # ptime /opt/schily/bin/star -x -xdir -xdot -U -fs=64m -fifostats -time file=../linux-3.5.1.tar
star: fifo had 46849 puts 80626 gets.
star: fifo was 1 times empty and 19 times full.
star: fifo held 67110912 bytes max, size was 67112960 bytes
star: fifo had 5 moves, total of 25600 moved bytes
star: fifo is 0% full (2k), size 65540k.
star: 46849 blocks + 0 bytes (total of 479733760 bytes = 468490.00k).
star: Total time 930.722sec (503 kBytes/sec)

real    15:31.002
user        1.802
sys        29.512

jupiter-sparc-SunOS5.10 # ptime rm -rf foo 

real     2:15.059
user        0.502
sys         7.177

jupiter-sparc-SunOS5.10 # mkdir doo
jupiter-sparc-SunOS5.10 # cd doo
jupiter-sparc-SunOS5.10 # ptime /usr/sfw/bin/gtar -xf ../linux-3.5.1.tar

real     3:57.001
user        2.065
sys        18.466
jupiter-sparc-SunOS5.10 #

So there we have some numbers. 

Dennis