Gigabyte GA-MA74GM-S2H - FreeBSD - 2015 archive

back to main FreeBSD page for this machine.

2015 archive.

2015-12-12: latest scrub result, storage:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h22m with 0 errors on Sat Dec 12 16:05:56 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

nothing more.

2015-12-11: latest scrub result, zroot:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Fri Dec 11 16:12:56 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2015-11-14: change to new firewall noted in /var/log/messages:

Nov 14 13:48:26 kg-f2 kernel: arp: 10.1.10.1 moved from 00:10:4b:e3:38:8b to 80:ee:73:60:61:0c on re0

nothing more.

2015-11-01: latest scrub status of storage pool:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h22m with 0 errors on Sun Nov  1 16:14:52 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2015-11-01: latest scrub status of zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Sun Nov  1 12:48:30 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2015-10-04: latest scrub status of storage pool:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h22m with 0 errors on Sun Oct  4 15:38:55 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2015-10-02: latest scrub status of zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Fri Oct  2 09:03:09 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2015-09-04: latest scrub status of storage pool:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h22m with 0 errors on Fri Sep  4 13:13:07 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2015-09-01: latest scrub status of zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Tue Sep  1 16:13:59 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2015-05-15: latest scrub status of zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Fri May 15 19:17:13 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

all for now.

2015-05-09: latest scrub status of storage pool:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h22m with 0 errors on Sat May  9 14:45:38 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

nothing more.

2015-04-06: latest scrub status of storage pool:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h22m with 0 errors on Mon Apr  6 21:56:19 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

all I had for now.

2015-04-02: latest scrub status of zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Thu Apr  2 12:05:27 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

nothing more.

2015-03-17: latest status of zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Tue Mar 17 20:29:15 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

that's all.

2015-03-04: status - storage pool after resilvering:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: resilver completed after 2h48m with 0 errors on Wed Mar  4 01:15:24 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0  873G resilvered

errors: No known data errors

that's it.

2015-03-03: replace hard drive ada0. First try to offline it:

root@kg-f2# zpool offline storage ada0

did it work?

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices has been taken offline by the administrator.
    Sufficient replicas exist for the pool to continue functioning in a
    degraded state.
action: Online the device using 'zpool online' or replace the device with
    'zpool replace'.
 scrub: scrub completed after 307445734561825859h20m with 0 errors on Mon Mar  2 21:40:47 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     DEGRADED     0     0     0
      raidz1    DEGRADED     0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    OFFLINE      0     0     0

errors: No known data errors

yep. Next, replace the drive:

old drive - serial number: S246J1KSB01867 (labeled "REV. A", manufactured 2009.11)
new drive - serial number: S246J90Z334960 (labeled "REV. A", manufactured 2010.03)

after replacement, the drive spins up and the drive LED turns green again.

and the drive shows up in /var/log/messages:

root@kg-f2# tail -6 /var/log/messages
Mar  3 04:07:03 kg-f2 ntpd[1373]: kernel time sync status change 2001
Mar  3 22:14:59 kg-f2 kernel: ada0 at siisch0 bus 0 scbus0 target 0 lun 0
Mar  3 22:14:59 kg-f2 kernel: ada0: <SAMSUNG HD103SJ 1AJ100E4> ATA-8 SATA 2.x device
Mar  3 22:14:59 kg-f2 kernel: ada0: 300.000MB/s transfers (SATA 2.x, UDMA6, PIO 8192bytes)
Mar  3 22:14:59 kg-f2 kernel: ada0: Command Queueing enabled
Mar  3 22:14:59 kg-f2 kernel: ada0: 953869MB (1953525168 512 byte sectors: 16H 63S/T 16383C)

nice - reboot not needed. camcontrol output:

root@kg-f2# camcontrol devlist
<SAMSUNG HD103SJ 1AJ100E4>         at scbus0 target 0 lun 0 (ada0,pass0)

for good measure, smartctl output:

root@kg-f2# smartctl -i /dev/ada0
smartctl 5.40 2010-10-16 r3189 [FreeBSD 8.2-STABLE amd64] (local build)
Copyright (C) 2002-10 by Bruce Allen, http://smartmontools.sourceforge.net

=== START OF INFORMATION SECTION ===
Model Family:     SAMSUNG SpinPoint F3 series
Device Model:     SAMSUNG HD103SJ
Serial Number:    S246J90Z334960
Firmware Version: 1AJ100E4
User Capacity:    1,000,204,886,016 bytes
Device is:        In smartctl database [for details use: -P show]
ATA Version is:   8
ATA Standard is:  ATA-8-ACS revision 6
Local Time is:    Tue Mar  3 22:21:34 2015 CET
SMART support is: Available - device has SMART capability.
SMART support is: Disabled
hmm, why is SMART disabled? Anyway, just enable it:
root@kg-f2# smartctl -s on /dev/ada0
smartctl 5.40 2010-10-16 r3189 [FreeBSD 8.2-STABLE amd64] (local build)
Copyright (C) 2002-10 by Bruce Allen, http://smartmontools.sourceforge.net

=== START OF ENABLE/DISABLE COMMANDS SECTION ===
SMART Enabled.

ok. Next replace the drive in the storage pool:

root@kg-f2# zpool replace storage ada0 ada0

(this takes a few seconds)

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices is currently being resilvered.  The pool will
    continue to function, possibly in a degraded state.
action: Wait for the resilver to complete.
 scrub: resilver in progress for 0h0m, 0.39% done, 2h55m to go
config:

    NAME            STATE     READ WRITE CKSUM
    storage         DEGRADED     0     0     0
      raidz1        DEGRADED     0     0     0
        ad8         ONLINE       0     0     0
        ad10        ONLINE       0     0     0
        ad12        ONLINE       0     0     0
        ad14        ONLINE       0     0     0
        replacing   DEGRADED     0     0     0
          ada0/old  OFFLINE      0     0     0
          ada0      ONLINE       0     0     0  3.39G resilvered

errors: No known data errors

and the pool has started resilvering. Nice.

2015-03-02: a visual check gives a red LED for the rightmost drive in the hotswap bays. I bet that it is the ada0 drive.

2015-03-02: status of storage pool (after a manual scrub):

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices could not be opened.  Sufficient replicas exist for
    the pool to continue functioning in a degraded state.
action: Attach the missing device and online it using 'zpool online'.
   see: http://www.sun.com/msg/ZFS-8000-2Q
 scrub: scrub completed after 307445734561825859h20m with 0 errors on Mon Mar  2 21:40:47 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     DEGRADED     0     0     0
      raidz1    DEGRADED     0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    UNAVAIL      0     0     0  cannot open

errors: No known data errors

hmm atacontrol says:

root@kg-f2# atacontrol list
ATA channel 0:
    Master:      no device present
    Slave:       no device present
ATA channel 2:
    Master:  ad4 <SAMSUNG HD252HJ/1AC01118> SATA revision 2.x
    Slave:       no device present
ATA channel 3:
    Master:  ad6 <SAMSUNG HD252HJ/1AC01118> SATA revision 2.x
    Slave:       no device present
ATA channel 4:
    Master:  ad8 <SAMSUNG HD103SJ/1AJ100E4> SATA revision 2.x
    Slave:       no device present
ATA channel 5:
    Master: ad10 <SAMSUNG HD103SJ/1AJ100E4> SATA revision 2.x
    Slave:       no device present
ATA channel 6:
    Master: ad12 <SAMSUNG HD103SJ/1AJ100E4> SATA revision 2.x
    Slave:       no device present
ATA channel 7:
    Master: ad14 <SAMSUNG HD103SJ/1AJ100E4> SATA revision 2.x
    Slave:       no device present

and camcontrol says:

root@kg-f2# camcontrol devlist

How about /var/log/messages?

Mar  2 18:16:47 kg-f2 smartd[1301]: Device: /dev/ada0, 143 Currently unreadable (pending) sectors (changed +1)

Mar  2 19:46:47 kg-f2 smartd[1301]: Device: /dev/ada0, 144 Currently unreadable (pending) sectors (changed +1)
Mar  2 20:16:46 kg-f2 smartd[1301]: Device: /dev/ada0, 145 Currently unreadable (pending) sectors (changed +1)

then

Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 3fe00000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 28
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 2fe00000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 25
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 2de00000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 24
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 2ce00000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 27
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 24e00000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 23
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 24600000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 29
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 04600000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 22
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 04200000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 26
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:26:55 kg-f2 kernel: siisch0:  ... waiting for slots 00200000
Mar  2 21:26:55 kg-f2 kernel: siisch0: Timeout on slot 21
Mar  2 21:26:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:27:25 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:27:25 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 40000000 rs 40000000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:27:55 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:27:55 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 40000000 rs 40000000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:27:55 kg-f2 kernel: (ada0:siisch0:0:0:0): lost device

(here we lost ada0)

Mar  2 21:28:25 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:28:25 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 40000000 rs 40000000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:28:40 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:28:40 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:28:55 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:28:55 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:29:25 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:29:25 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 40000000 rs 40000000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:29:40 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:29:40 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:29:56 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:29:56 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 3fe00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 29
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 1fe00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 28
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 0fe00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 27
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 07e00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 26
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 03e00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 25
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 01e00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 24
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 00e00000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 23
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 00600000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 22
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:26 kg-f2 kernel: siisch0:  ... waiting for slots 00200000
Mar  2 21:30:26 kg-f2 kernel: siisch0: Timeout on slot 21
Mar  2 21:30:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:30:41 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:30:41 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:30:56 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:30:56 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 3fe00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 29
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 1fe00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 28
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 0fe00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 27
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 07e00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 26
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 03e00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 25
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 01e00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 24
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 00e00000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 23
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 00600000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 22
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:26 kg-f2 kernel: siisch0:  ... waiting for slots 00200000
Mar  2 21:31:26 kg-f2 kernel: siisch0: Timeout on slot 21
Mar  2 21:31:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:31:41 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:31:41 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:31:56 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:31:56 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 3fe00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 29
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 1fe00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 28
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 0fe00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 27
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 07e00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 26
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 03e00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 25
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 01e00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 24
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 00e00000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 23
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 00600000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 22
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:26 kg-f2 kernel: siisch0:  ... waiting for slots 00200000
Mar  2 21:32:26 kg-f2 kernel: siisch0: Timeout on slot 21
Mar  2 21:32:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:32:41 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:32:41 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:32:56 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:32:56 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:33:26 kg-f2 root: ZFS: vdev I/O failure, zpool=storage path=/dev/ada0 offset=270336 size=8192 error=6

(zfs notices I/O failure here)

Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 3fe00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 29
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 1fe00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 28
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 0fe00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 27
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 07e00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 26
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 03e00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 25
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 01e00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 24
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 00e00000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 23
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 00600000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 22
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 kernel: siisch0:  ... waiting for slots 00200000
Mar  2 21:33:26 kg-f2 kernel: siisch0: Timeout on slot 21
Mar  2 21:33:26 kg-f2 kernel: siisch0: siis_timeout is 00000000 ss 7fe00000 rs 7fe00000 es 00000000 sts 801f0000 serr 00000000
Mar  2 21:33:26 kg-f2 root: ZFS: vdev I/O failure, zpool=storage path=/dev/ada0 offset=1000204148736 size=8192 error=6
Mar  2 21:33:26 kg-f2 root: ZFS: vdev I/O failure, zpool=storage path=/dev/ada0 offset=1000204410880 size=8192 error=6

(more ZFS reactions)

Mar  2 21:33:41 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:33:41 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:33:56 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:33:56 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000

and a few minutes later, the machine reboots:

Mar  2 21:37:09 kg-f2 syslogd: kernel boot file is /boot/kernel/kernel
Mar  2 21:37:09 kg-f2 kernel: Copyright (c) 1992-2011 The FreeBSD Project.
Mar  2 21:37:09 kg-f2 kernel: Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994
Mar  2 21:37:09 kg-f2 kernel: The Regents of the University of California. All rights reserved.
Mar  2 21:37:09 kg-f2 kernel: FreeBSD is a registered trademark of The FreeBSD Foundation.
Mar  2 21:37:09 kg-f2 kernel: FreeBSD 8.2-STABLE #5: Fri Jun  3 17:20:39 CEST 2011
Mar  2 21:37:09 kg-f2 kernel: root@kg-f2.kg4.no:/usr/obj/usr/src/sys/GENERIC amd64

and there still is timeouts on the siis controller

Mar  2 21:37:09 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:37:09 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:37:09 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:37:09 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:37:09 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:37:09 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000
Mar  2 21:37:09 kg-f2 kernel: siisch0: Timeout on slot 30
Mar  2 21:37:09 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000000 sts 801e0000 serr 00000000

time to replace ada0 me thinks.

root@kg-f2# uptime
11:11PM  up  1:36, 1 user, load averages: 0.00, 0.00, 0.00

nothing more.

2015-03-02: latest scrub result on zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Mon Mar  2 17:45:53 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

that's it.

2015-01-18: since everything seems to be in order, let's clear the error on the storage pool:

root@kg-f2# zpool clear storage

verify:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
 scrub: scrub completed after 3h46m with 0 errors on Sun Jan 18 16:30:43 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     0  376K repaired

errors: No known data errors

done.

2015-01-18: latest scrub result on storage pool:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices has experienced an unrecoverable error.  An
    attempt was made to correct the error.  Applications are unaffected.
action: Determine if the device needs to be replaced, and clear the errors
    using 'zpool clear' or replace the device with 'zpool replace'.
   see: http://www.sun.com/msg/ZFS-8000-9P
 scrub: scrub completed after 3h46m with 0 errors on Sun Jan 18 16:30:43 2015
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1    ONLINE       0     0     0
        ad8     ONLINE       0     0     0
        ad10    ONLINE       0     0     0
        ad12    ONLINE       0     0     0
        ad14    ONLINE       0     0     0
        ada0    ONLINE       0     0     4  376K repaired

errors: No known data errors

also, /var/log/messages has these lines:

Jan 18 14:28:54 kg-f2 root: ZFS: checksum mismatch, zpool=storage path=/dev/ada0 offset=334592438784 size=1536
Jan 18 14:46:47 kg-f2 smartd[1301]: Device: /dev/ada0, 128 Currently unreadable (pending) sectors
Jan 18 15:16:49 kg-f2 smartd[1301]: Device: /dev/ada0, 129 Currently unreadable (pending) sectors (changed +1)
Jan 18 15:46:47 kg-f2 smartd[1301]: Device: /dev/ada0, 128 Currently unreadable (pending) sectors (changed -1)

Jan 18 16:10:26 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:10:27 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:10:45 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:10:46 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:11:29 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:11:30 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:11:32 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:11:33 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:11:37 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:11:38 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:12:08 kg-f2 kernel: siisch0: Timeout on slot 30
Jan 18 16:12:08 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000005 sts 001f0000 serr 00000000
Jan 18 16:12:27 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:12:28 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
[...]
Jan 18 16:16:04 kg-f2 kernel: siisch0: Timeout on slot 30
Jan 18 16:16:04 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 60000000 rs 60000000 es 00000005 sts 001f0000 serr 00000000
Jan 18 16:16:04 kg-f2 kernel: siisch0:  ... waiting for slots 20000000
Jan 18 16:16:04 kg-f2 kernel: siisch0: Timeout on slot 29
Jan 18 16:16:04 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 60000000 rs 60000000 es 00000005 sts 001f0000 serr 00000000
Jan 18 16:16:06 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:16:06 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000

Jan 18 16:16:47 kg-f2 smartd[1301]: Device: /dev/ada0, failed to read SMART Attribute Data

Jan 18 16:26:31 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:26:32 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:26:56 kg-f2 root: ZFS: checksum mismatch, zpool=storage path=/dev/ada0 offset=915155059200 size=32768
Jan 18 16:26:57 kg-f2 root: ZFS: checksum mismatch, zpool=storage path=/dev/ada0 offset=915157942784 size=32768
Jan 18 16:27:04 kg-f2 root: ZFS: checksum mismatch, zpool=storage path=/dev/ada0 offset=915173240832 size=32768
Jan 18 16:27:57 kg-f2 kernel: siisch0: Error while READ LOG EXT
Jan 18 16:27:57 kg-f2 kernel: siisch0: port is not ready (timeout 1000ms) status = 001f0000
Jan 18 16:28:28 kg-f2 kernel: siisch0: Timeout on slot 30
Jan 18 16:28:28 kg-f2 kernel: siisch0: siis_timeout is 00040000 ss 40000000 rs 40000000 es 00000005 sts 001f0000 serr 00000000
Jan 18 16:46:45 kg-f2 smartd[1301]: Device: /dev/ada0, 137 Currently unreadable (pending) sectors (changed +9)

smartctl still says:

root@kg-f2# smartctl -H /dev/ada0
smartctl 5.40 2010-10-16 r3189 [FreeBSD 8.2-STABLE amd64] (local build)
Copyright (C) 2002-10 by Bruce Allen, http://smartmontools.sourceforge.net

=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED

ok.

2015-01-18: latest scrub result on zroot pool:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
 scrub: scrub completed after 0h3m with 0 errors on Sun Jan 18 12:38:06 2015
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror       ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

nothing more.