Gigabyte GA-MA74GM-S2H - FreeBSD - 2018 archive

back to main FreeBSD page for this machine.

2018 archive.

2018-12-26: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h41m with 0 errors on Wed Dec 26 06:47:55 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-12-09: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Sun Dec  9 03:07:16 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-11-20: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h47m with 0 errors on Tue Nov 20 07:40:08 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-11-03: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Sat Nov  3 03:41:33 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-10-16: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h49m with 0 errors on Tue Oct 16 07:38:36 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-10-06: ports - installed curl

Oct  7 15:29:39 kg-f2 pkg-static: ca_root_nss-3.22 installed
Oct  7 15:29:43 kg-f2 pkg-static: curl-7.47.0 installed

ok.

2018-09-29: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Sat Sep 29 04:01:22 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-09-10: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h45m with 0 errors on Mon Sep 10 06:48:32 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok

2018-08-24: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Fri Aug 24 03:06:58 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-08-05: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h40m with 0 errors on Sun Aug  5 07:30:19 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-07-19: zpool storage - resilver completed

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
  scan: resilvered 875G in 31h26m with 0 errors on Thu Jul 19 01:19:58 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-07-17: ada6 - replace it status before

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices has been removed by the administrator.
    Sufficient replicas exist for the pool to continue functioning in a
    degraded state.
action: Online the device using 'zpool online' or replace the device with
    'zpool replace'.
  scan: resilvered 0 in 0h0m with 0 errors on Mon Jul 16 21:51:58 2018
config:

    NAME                      STATE     READ WRITE CKSUM
    storage                   DEGRADED     0     0     0
      raidz1-0                DEGRADED     0     0     0
        ada3                  ONLINE       0     0     0
        ada4                  ONLINE       0     0     0
        ada5                  ONLINE       0     0     0
        17796649983653810714  REMOVED      0     0     0  was /dev/ada6
        ada0                  ONLINE       0     0     0

errors: No known data errors

replace

root@kg-f2# zpool replace storage ada6
(it chews on it for a few seconds)

status after

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices is currently being resilvered.  The pool will
    continue to function, possibly in a degraded state.
action: Wait for the resilver to complete.
  scan: resilver in progress since Tue Jul 17 17:53:51 2018
        5.18G scanned out of 4.27T at 32.7M/s, 37h59m to go
        1.04G resilvered, 0.12% done
config:

    NAME                        STATE     READ WRITE CKSUM
    storage                     DEGRADED     0     0     0
      raidz1-0                  DEGRADED     0     0     0
        ada3                    ONLINE       0     0     0
        ada4                    ONLINE       0     0     0
        ada5                    ONLINE       0     0     0
        replacing-3             REMOVED      0     0     0
          17796649983653810714  REMOVED      0     0     0  was /dev/ada6/old
          ada6                  ONLINE       0     0     0  block size: 512B configured, 4096B native  (resilvering)
        ada0                    ONLINE       0     0     0

errors: No known data errors

ok. I'll remove the dangling reference if it still is there after resilvering finishes.

2018-07-17: ada6 - after replacing the cables, see if I can write to the drive:

root@kg-f2# dd if=/dev/zero of=/dev/ada6 bs=1M count=1
1+0 records in
1+0 records out
1048576 bytes transferred in 0.005373 secs (195156483 bytes/sec)

yes, no problems.

2018-07-16: replacing ada6 - check out the new drive

smartctl info
root@kg-f2# smartctl -i /dev/ada6
smartctl 6.4 2015-06-04 r4109 [FreeBSD 8.4-STABLE amd64] (local build)
Copyright (C) 2002-15, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===
Device Model:     TOSHIBA HDWD110
Serial Number:    483JS0WNS
LU WWN Device Id: 5 000039 fd6c79c62
Firmware Version: MS2OA8J0
User Capacity:    1,000,204,886,016 bytes [1.00 TB]
Sector Sizes:     512 bytes logical, 4096 bytes physical
Rotation Rate:    7200 rpm
Form Factor:      3.5 inches
Device is:        Not in smartctl database [for details use: -P showall]
ATA Version is:   ATA8-ACS T13/1699-D revision 4
SATA Version is:  SATA 3.0, 6.0 Gb/s (current: 3.0 Gb/s)
Local Time is:    Mon Jul 16 22:09:24 2018 CEST
SMART support is: Available - device has SMART capability.
SMART support is: Enabled

gpart info

root@kg-f2# gpart show -p ada6
gpart: No such geom: ada6.

nothing - good. See if we can write to the drive

root@kg-f2# dd if=/dev/zero of=/dev/ada6 bs=1M count=1
dd: /dev/ada6: Input/output error
1+0 records in
0+0 records out
0 bytes transferred in 0.416720 secs (0 bytes/sec)
root@kg-f2# dd if=/dev/zero of=/dev/ada6 bs=1M count=1
dd: /dev/ada6: Input/output error
1+0 records in
0+0 records out
0 bytes transferred in 0.518042 secs (0 bytes/sec)
root@kg-f2# dd if=/dev/zero of=/dev/ada6 bs=1M count=1
dd: /dev/ada6: Input/output error
1+0 records in
0+0 records out
0 bytes transferred in 0.416168 secs (0 bytes/sec)

no? Try unplug, replug - didn't help. Try this

root@kg-f2# gpart create -s GPT ada6
gpart: Input/output error
root@kg-f2# gpart create -s GPT ada6
gpart: geom 'ada6': File exists
root@kg-f2# gpart show -p ada6
=>        34  1953525101  ada6  GPT  (931G)
          34  1953525101        - free -  (931G)
root@kg-f2# gpart destroy ada6
ada6 destroyed
root@kg-f2# gpart show -p ada6
gpart: No such geom: ada6.
root@kg-f2# dd if=/dev/zero of=/dev/ada6 bs=1M count=1
dd: /dev/ada6: Input/output error
1+0 records in
0+0 records out
0 bytes transferred in 0.415921 secs (0 bytes/sec)

nope, didn't help. Perhaps I need a new sata cable.

2018-07-15: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices are faulted in response to persistent errors.
    Sufficient replicas exist for the pool to continue functioning in a
    degraded state.
action: Replace the faulted device, or use 'zpool clear' to mark the device
    repaired.
  scan: resilvered 263K in 0h0m with 0 errors on Sun Jul 15 02:11:08 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     DEGRADED     0     0     0
      raidz1-0  DEGRADED     0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    FAULTED      2 1.67M    36  too many errors
        ada0    ONLINE       0     0     0

errors: No known data errors

in /var/log/messages

Jul 14 15:09:01 kg-f2 kernel: ahcich5: Timeout on slot 28 port 0
Jul 14 15:09:01 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 30000000 rs 30000000 tfd 40 serr 00400000 cmd 0004fd17
Jul 14 16:21:05 kg-f2 kernel: ahcich5: Timeout on slot 19 port 0
Jul 14 16:21:05 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00780000 rs 00780000 tfd 40 serr 00400000 cmd 0004f617
Jul 14 17:35:51 kg-f2 smartd[681]: Device: /dev/ada6, failed to read SMART Attribute Data
Jul 14 17:37:55 kg-f2 kernel: ahcich5: Timeout on slot 17 port 0
Jul 14 17:37:55 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00020000 rs 00020000 tfd 40 serr 00400000 cmd 0004f117
Jul 14 17:42:27 kg-f2 kernel: ahcich5: Timeout on slot 23 port 0
Jul 14 17:42:27 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00800000 rs 00800000 tfd 40 serr 00400000 cmd 0004f717
Jul 14 17:46:51 kg-f2 kernel: ahcich5: Timeout on slot 21 port 0
Jul 14 17:46:51 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00200000 rs 00200000 tfd 40 serr 00400000 cmd 0004f517
Jul 14 18:35:52 kg-f2 smartd[681]: Device: /dev/ada6, failed to read SMART Attribute Data
Jul 14 18:45:46 kg-f2 kernel: ahcich5: Timeout on slot 17 port 0
Jul 14 18:45:46 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00020000 rs 00020000 tfd 40 serr 00400000 cmd 0004f117
Jul 14 19:07:59 kg-f2 kernel: ahcich5: Timeout on slot 18 port 0
Jul 14 19:07:59 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00040000 rs 00040000 tfd 40 serr 00400000 cmd 0004f217
Jul 14 19:24:01 kg-f2 kernel: ahcich5: Timeout on slot 3 port 0
Jul 14 19:24:01 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00000078 rs 00000078 tfd 40 serr 00400000 cmd 0004e617
Jul 14 20:05:52 kg-f2 smartd[681]: Device: /dev/ada6, failed to read SMART Attribute Data
Jul 14 20:10:26 kg-f2 kernel: ahcich5: Timeout on slot 9 port 0
Jul 14 20:10:26 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00001e00 rs 00001e00 tfd 40 serr 00400000 cmd 0004ec17
Jul 14 23:01:29 kg-f2 kernel: ahcich5: Timeout on slot 19 port 0
Jul 14 23:01:29 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00080000 rs 00080000 tfd 40 serr 00400000 cmd 0004f317
Jul 14 23:32:25 kg-f2 kernel: ahcich5: Timeout on slot 26 port 0
Jul 14 23:32:25 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 04000000 rs 04000000 tfd 40 serr 00400000 cmd 0004fa17
Jul 14 23:35:20 kg-f2 kernel: ahcich5: Timeout on slot 18 port 0
Jul 14 23:35:20 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00040000 rs 00040000 tfd 40 serr 00400000 cmd 0004f217
Jul 14 23:35:52 kg-f2 smartd[681]: Device: /dev/ada6, failed to read SMART Attribute Data
Jul 15 00:03:20 kg-f2 kernel: ahcich5: Timeout on slot 17 port 0
Jul 15 00:03:20 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00020000 rs 00020000 tfd 40 serr 00400000 cmd 0004f117
Jul 15 00:15:46 kg-f2 kernel: ahcich5: Timeout on slot 1 port 0
Jul 15 00:15:46 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 0000001e rs 0000001e tfd 40 serr 00400000 cmd 0004e417
Jul 15 00:35:00 kg-f2 kernel: ahcich5: Timeout on slot 4 port 0
Jul 15 00:35:00 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00000010 rs 00000010 tfd 40 serr 00400000 cmd 0004e417
Jul 15 01:08:20 kg-f2 kernel: ahcich5: Timeout on slot 25 port 0
Jul 15 01:08:20 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 1e000000 rs 1e000000 tfd 40 serr 00400000 cmd 0004fc17
Jul 15 01:46:53 kg-f2 kernel: ahcich5: Timeout on slot 29 port 0
Jul 15 01:46:53 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 20000000 rs 20000000 tfd 40 serr 00400000 cmd 0004fd17

smartctl on ada6

root@kg-f2# smartctl -H /dev/ada6
smartctl 6.4 2015-06-04 r4109 [FreeBSD 8.4-STABLE amd64] (local build)
Copyright (C) 2002-15, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED
and - smartctl -a says that there are no error logs. Strange.

clear

root@kg-f2# zpool clear storage

new status

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices has experienced an unrecoverable error.  An
    attempt was made to correct the error.  Applications are unaffected.
action: Determine if the device needs to be replaced, and clear the errors
    using 'zpool clear' or replace the device with 'zpool replace'.
   see: http://illumos.org/msg/ZFS-8000-9P
  scan: resilvered 878K in 0h0m with 0 errors on Sun Jul 15 14:46:09 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0 1.19K     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

maybe I should replace ada6 again

2018-07-13: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices has experienced an unrecoverable error.  An
    attempt was made to correct the error.  Applications are unaffected.
action: Determine if the device needs to be replaced, and clear the errors
    using 'zpool clear' or replace the device with 'zpool replace'.
   see: http://illumos.org/msg/ZFS-8000-9P
  scan: scrub repaired 1K in 4h15m with 0 errors on Fri Jul 13 07:16:29 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE      10 9.24M     1  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

in /var/log/messages

Jul 13 03:20:15 kg-f2 kernel: ahcich5: Timeout on slot 27 port 0
Jul 13 03:20:15 kg-f2 kernel: ahcich5: is 04000000 cs 18000000 ss 18000000 rs 18000000 tfd 40 serr 00400000 cmd 0004fb17
Jul 13 04:27:49 kg-f2 kernel: ahcich5: Timeout on slot 3 port 0
Jul 13 04:27:49 kg-f2 kernel: ahcich5: is 04000000 cs 00000000 ss 00000008 rs 00000008 tfd 40 serr 00400000 cmd 0004e317
Jul 13 05:41:08 kg-f2 kernel: ahcich5: Timeout on slot 24 port 0
Jul 13 05:41:08 kg-f2 kernel: ahcich5: is 04000000 cs 03000000 ss 03000000 rs 03000000 tfd 40 serr 00400001 cmd 0004f817
Jul 13 06:24:30 kg-f2 kernel: ahcich5: Timeout on slot 31 port 0
Jul 13 06:24:30 kg-f2 kernel: ahcich5: is 04000000 cs 80000001 ss 80000001 rs 80000001 tfd 40 serr 00400000 cmd 0004ff17
Jul 13 06:51:54 kg-f2 kernel: ahcich5: Timeout on slot 30 port 0
Jul 13 06:51:54 kg-f2 kernel: ahcich5: is 04000000 cs 40000000 ss 40000000 rs 40000000 tfd 40 serr 00400001 cmd 0004fe17

smartctl on ada6

root@kg-f2# smartctl -H /dev/ada6
smartctl 6.4 2015-06-04 r4109 [FreeBSD 8.4-STABLE amd64] (local build)
Copyright (C) 2002-15, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED

clear

root@kg-f2# zpool clear storage

new status

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 1K in 4h15m with 0 errors on Fri Jul 13 07:16:29 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-07-01: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices has experienced an unrecoverable error.  An
    attempt was made to correct the error.  Applications are unaffected.
action: Determine if the device needs to be replaced, and clear the errors
    using 'zpool clear' or replace the device with 'zpool replace'.
   see: http://illumos.org/msg/ZFS-8000-9P
  scan: scrub repaired 90.5K in 3h58m with 0 errors on Sun Jul  1 07:22:20 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0    74  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

in /var/log/messages

Jul  1 06:33:40 kg-f2 kernel: ahcich5: Timeout on slot 24 port 0
Jul  1 06:33:40 kg-f2 kernel: ahcich5: is 00000000 cs 03000000 ss 03000000 rs 03000000 tfd 40 serr 00400001 cmd 0004f817
Jul  1 06:40:21 kg-f2 kernel: ahcich5: Timeout on slot 20 port 0
Jul  1 06:40:21 kg-f2 kernel: ahcich5: is 04000000 cs 00300000 ss 00300000 rs 00300000 tfd 40 serr 00400001 cmd 0004f417
Jul  1 06:51:18 kg-f2 kernel: ahcich5: Timeout on slot 9 port 0
Jul  1 06:51:18 kg-f2 kernel: ahcich5: is 04000000 cs 00000600 ss 00000600 rs 00000600 tfd 40 serr 00400001 cmd 0004e917
Jul  1 06:55:08 kg-f2 kernel: ahcich5: Timeout on slot 24 port 0
Jul  1 06:55:08 kg-f2 kernel: ahcich5: is 04000000 cs 03000000 ss 03000000 rs 03000000 tfd 40 serr 00400001 cmd 0004f817

smartctl says

root@kg-f2# smartctl -H /dev/ada6
smartctl 6.4 2015-06-04 r4109 [FreeBSD 8.4-STABLE amd64] (local build)
Copyright (C) 2002-15, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED

fix?

root@kg-f2# zpool clear storage

new status

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 90.5K in 3h58m with 0 errors on Sun Jul  1 07:22:20 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-06-13: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Wed Jun 13 03:37:48 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-05-26: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h34m with 0 errors on Sat May 26 06:37:23 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-05-08: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Tue May  8 03:38:23 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-04-20: scrub - latest result for pool storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h33m with 0 errors on Fri Apr 20 07:33:12 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-04-03: scrub - latest result for pool zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Tue Apr  3 03:07:04 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-03-15: scrub - latest result for pool storage:

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h44m with 0 errors on Thu Mar 15 07:14:40 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-02-26: scrub - latest result for pool zroot:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Mon Feb 26 03:46:27 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

2018-02-18: booted the server after replacing the PSU. status

root@kg-f2# date;temp;swapinfo -h;echo " ";tvlm;echo " ";tvls;echo " ";df -h;echo " ";uptime;echo " ";ps ax | grep -v grep | grep ddc;ps ax | grep -v grep | grep smartd;ps ax | grep -v grep | grep natd
Sun Feb 18 16:47:18 CET 2018
dev.cpu.0.temperature: 43.0C
dev.cpu.1.temperature: 43.0C
Device          1K-blocks     Used    Avail Capacity
/dev/gpt/swap0    4194304       0B     4.0G     0%
/dev/gpt/swap1    4194304       0B     4.0G     0%
Total             8388608       0B     8.0G     0%

Feb 18 16:21:28 kg-f2 kernel: uhub5: 6 ports with 6 removable, self powered
Feb 18 16:21:28 kg-f2 kernel: Trying to mount root from zfs:zroot
Feb 18 16:21:29 kg-f2 ntpd[629]: ntpd 4.2.4p5-a (1)
Feb 18 16:21:31 kg-f2 kernel: re0: link state changed to UP
Feb 18 16:21:33 kg-f2 dhclient: New IP Address (re0): 10.1.150.18
Feb 18 16:21:33 kg-f2 dhclient: New Subnet Mask (re0): 255.255.0.0
Feb 18 16:21:33 kg-f2 dhclient: New Broadcast Address (re0): 10.1.255.255
Feb 18 16:21:33 kg-f2 dhclient: New Routers (re0): 10.1.10.1
Feb 18 16:25:57 kg-f2 ntpd[630]: time reset +7.201847 s
Feb 18 16:31:30 kg-f2 su: tingo to root on /dev/pts/0

Dec 10 10:09:29 kg-f2 newsyslog[570]: logfile first created

Filesystem                  Size    Used   Avail Capacity  Mounted on
zroot                        54G    859M     53G     2%    /
devfs                       1.0k    1.0k      0B   100%    /dev
storage                     3.6T    3.4T    161G    96%    /storage
zroot/tmp                    53G    408k     53G     0%    /tmp
zroot/usr                    55G    2.5G     53G     5%    /usr
zroot/usr/home               53G    599k     53G     0%    /usr/home
zroot/usr/ports              54G    765M     53G     1%    /usr/ports
zroot/usr/ports/packages     53G     30k     53G     0%    /usr/ports/packages
zroot/usr/src                54G    1.2G     53G     2%    /usr/src
zroot/var                    53G    9.2M     53G     0%    /var
zroot/var/crash              53G     22k     53G     0%    /var/crash
zroot/var/db                 53G    107M     53G     0%    /var/db
zroot/var/db/pkg             53G    8.5M     53G     0%    /var/db/pkg
zroot/var/empty              53G     21k     53G     0%    /var/empty
zroot/var/log                53G    577k     53G     0%    /var/log
zroot/var/mail               53G     31k     53G     0%    /var/mail
zroot/var/run                53G     67k     53G     0%    /var/run
zroot/var/tmp                53G    1.9M     53G     0%    /var/tmp

 4:47PM  up 26 mins, 1 user, load averages: 1.02, 0.96, 0.85

  693  ??  I      0:00.00 /usr/local/sbin/smartd -c /usr/local/etc/smartd.conf

zpool zroot status

root@kg-f2# zpool status zroot                                          
 pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Sun Jan 21 03:25:05 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

zpool storage status

root@kg-f2# zpool status storage                                            
 pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub in progress since Wed Feb  7 03:17:58 2018
        676G scanned out of 4.26T at 1/s, (scan is slow, no estimated time)
        0 repaired, 15.48% done
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.

2018-02-03: no lost devices this time:

Feb  2 17:52:55 kg-f2 su: tingo to root on /dev/pts/0
Feb  3 03:03:29 kg-f2 syslogd: kernel boot file is /boot/kernel/kernel
Feb  3 03:03:29 kg-f2 kernel: Copyright (c) 1992-2015 The FreeBSD Project.

Feb  3 03:05:51 kg-f2 syslogd: kernel boot file is /boot/kernel/kernel
Feb  3 03:05:51 kg-f2 kernel: Copyright (c) 1992-2015 The FreeBSD Project.

last info

root@kg-f2# date;last
Sun Feb  4 13:46:22 CET 2018
tingo            pts/0    kg-core1.kg4.no  Sat Feb  3 14:16   still logged in
reboot           ~                         Sat Feb  3 03:06
reboot           ~                         Sat Feb  3 03:03
tingo            pts/0    kg-core1.kg4.no  Fri Feb  2 17:52 - crash  (09:10)
reboot           ~                         Fri Feb  2 03:04

wtmp begins Fri Feb  2 03:04:30 CET 2018

perhaps I should replace the PSU.

2018-02-02: interesting from /var/log/messages:

Jan 27 13:03:52 kg-f2 su: tingo to root on /dev/pts/1
Feb  2 03:02:50 kg-f2 kernel: (ada6:ahcich5:0:0:0): lost device
Feb  2 03:04:19 kg-f2 syslogd: kernel boot file is /boot/kernel/kernel
Feb  2 03:04:19 kg-f2 kernel: Copyright (c) 1992-2015 The FreeBSD Project.

looks like the machine rebooted when ada6 was lost.

last info

root@kg-f2# last
tingo            pts/0    kg-core1.kg4.no  Fri Feb  2 17:52   still logged in
reboot           ~                         Fri Feb  2 03:04

wtmp begins Fri Feb  2 03:04:30 CET 2018

yep, reboot confirmed

zpool status

zroot

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Sun Jan 21 03:25:05 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

storage

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: resilvered 328K in 0h0m with 0 errors on Fri Jan 19 16:53:37 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

pools ok.

2018-01-21: scrub - latest result for pool zroot:

root@kg-f2# zpool status zroot
  pool: zroot
 state: ONLINE
  scan: scrub repaired 0 in 0h4m with 0 errors on Sun Jan 21 03:25:05 2018
config:

    NAME           STATE     READ WRITE CKSUM
    zroot          ONLINE       0     0     0
      mirror-0     ONLINE       0     0     0
        gpt/disk0  ONLINE       0     0     0
        gpt/disk1  ONLINE       0     0     0

errors: No known data errors

ok.

root@kg-f2# smartctl -H /dev/ada6
smartctl 6.4 2015-06-04 r4109 [FreeBSD 8.4-STABLE amd64] (local build)
Copyright (C) 2002-15, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF READ SMART DATA SECTION ===
SMART overall-health self-assessment test result: PASSED

ada6 looks ok too.

2018-01-19: zpool status for storage

root@kg-f2# zpool status storage
  pool: storage
 state: DEGRADED
status: One or more devices has been removed by the administrator.
    Sufficient replicas exist for the pool to continue functioning in a
    degraded state.
action: Online the device using 'zpool online' or replace the device with
    'zpool replace'.
  scan: resilvered 339K in 0h0m with 0 errors on Wed Jan 17 03:10:35 2018
config:

    NAME                      STATE     READ WRITE CKSUM
    storage                   DEGRADED     0     0     0
      raidz1-0                DEGRADED     0     0     0
        ada3                  ONLINE       0     0     0
        ada4                  ONLINE       0     0     0
        ada5                  ONLINE       0     0     0
        17796649983653810714  REMOVED      0     0     0  was /dev/ada6
        ada0                  ONLINE       0     0     0

errors: No known data errors

hmm, what does history show

root@kg-f2# zpool history storage | tail -5
2017-11-05.15:52:08 zpool offline storage ada6
2017-11-06.21:50:00 zpool replace storage ada6
2017-11-29.03:02:46 zpool scrub storage
2018-01-03.03:08:53 zpool scrub storage

ok, so not done intentionally. Is the device available?

root@kg-f2# ls -l /dev/ada6
crw-r-----  1 root  operator    0, 104 Jan 17 07:19 /dev/ada6

ok, can I online it?

root@kg-f2# zpool online storage ada6

apparently. New status

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: resilvered 328K in 0h0m with 0 errors on Fri Jan 19 16:53:37 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok. Looks all right to me.

2018-01-17: interesting messages

Jan 17 05:43:54 kg-f2 kernel: (ada6:ahcich5:0:0:0): lost device
Jan 17 05:43:54 kg-f2 kernel: (ada6:ahcich5:0:0:0): removing device entry
Jan 17 05:43:57 kg-f2 kernel: ada6 at ahcich5 bus 0 scbus6 target 0 lun 0
Jan 17 05:43:57 kg-f2 kernel: ada6: <WDC WD10EZEX-00RKKA0 80.00A80> ATA-8 SATA 3.x device
Jan 17 05:43:57 kg-f2 kernel: ada6: 300.000MB/s transfers (SATA 2.x, UDMA6, PIO 8192bytes)
Jan 17 05:43:57 kg-f2 kernel: ada6: Command Queueing enabled
Jan 17 05:43:57 kg-f2 kernel: ada6: 953869MB (1953525168 512 byte sectors: 16H 63S/T 16383C)
Jan 17 07:19:15 kg-f2 kernel: (ada6:ahcich5:0:0:0): lost device
Jan 17 07:19:15 kg-f2 kernel: (ada6:ahcich5:0:0:0): removing device entry
Jan 17 07:19:18 kg-f2 kernel: ada6 at ahcich5 bus 0 scbus6 target 0 lun 0
Jan 17 07:19:18 kg-f2 kernel: ada6: <WDC WD10EZEX-00RKKA0 80.00A80> ATA-8 SATA 3.x device
Jan 17 07:19:18 kg-f2 kernel: ada6: 300.000MB/s transfers (SATA 2.x, UDMA6, PIO 8192bytes)
Jan 17 07:19:18 kg-f2 kernel: ada6: Command Queueing enabled
Jan 17 07:19:18 kg-f2 kernel: ada6: 953869MB (1953525168 512 byte sectors: 16H 63S/T 16383C)

ok.

2018-01-03: scrub - latest result for storage pool

root@kg-f2# zpool status storage
  pool: storage
 state: ONLINE
status: One or more devices are configured to use a non-native block size.
    Expect reduced performance.
action: Replace affected devices with devices that support the
    configured block size, or migrate data to a properly configured
    pool.
  scan: scrub repaired 0 in 3h33m with 0 errors on Wed Jan  3 06:42:13 2018
config:

    NAME        STATE     READ WRITE CKSUM
    storage     ONLINE       0     0     0
      raidz1-0  ONLINE       0     0     0
        ada3    ONLINE       0     0     0
        ada4    ONLINE       0     0     0
        ada5    ONLINE       0     0     0
        ada6    ONLINE       0     0     0  block size: 512B configured, 4096B native
        ada0    ONLINE       0     0     0

errors: No known data errors

ok.