# mdadm fails to add a spare to a raid1 array

## sinanqapudan

Now, this is driving me crazy:

```
sargon ~ # ls /dev

autofs           log                 sda   stderr  tty21  tty38  tty54  urandom  vcs8

block            mcelog              sda1  stdin   tty22  tty39  tty55  usb      vcs9

bsg              md                  sdb   stdout  tty23  tty4   tty56  usbmon0  vcsa

bus              md0                 sdb1  tty     tty24  tty40  tty57  usbmon1  vcsa1

char             md126               sdb2  tty0    tty25  tty41  tty58  usbmon2  vcsa10

console          md127               sdb3  tty1    tty26  tty42  tty59  usbmon3  vcsa11

core             mem                 sdc   tty10   tty27  tty43  tty6   usbmon4  vcsa12

cpu              mqueue              sdc1  tty11   tty28  tty44  tty60  vcs      vcsa2

cpu_dma_latency  network_latency     sdc2  tty12   tty29  tty45  tty61  vcs1     vcsa3

disk             network_throughput  sdc3  tty13   tty3   tty46  tty62  vcs10    vcsa4

fd               null                sdd   tty14   tty30  tty47  tty63  vcs11    vcsa5

full             pktcdvd             sdd1  tty15   tty31  tty48  tty7   vcs12    vcsa6

hidraw0          port                sde   tty16   tty32  tty49  tty8   vcs2     vcsa7

hidraw1          psaux               sdf   tty17   tty33  tty5   tty9   vcs3     vcsa8

hpet             ptmx                sdf1  tty18   tty34  tty50  ttyS0  vcs4     vcsa9

initctl          pts                 sdg   tty19   tty35  tty51  ttyS1  vcs5     vga_arbiter

input            random              sdg1  tty2    tty36  tty52  ttyS2  vcs6     zero

kmsg             root                shm   tty20   tty37  tty53  ttyS3  vcs7

sargon ~ # cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdb3[0]

      157885888 blocks [2/1] [U_]

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # mdadm -E /dev/sdc

/dev/sdc:

   MBR Magic : aa55

Partition[0] :       204800 sectors at         2048 (type fd)

Partition[1] :      4194304 sectors at       206848 (type 82)

Partition[2] :    315771904 sectors at      4401152 (type fd)

sargon ~ # mdadm -E /dev/sd*

/dev/sda:

   MBR Magic : aa55

Partition[0] :   1953523120 sectors at         2048 (type 83)

mdadm: No md superblock detected on /dev/sda1.

/dev/sdb:

          Magic : a92b4efc

        Version : 0.90.00

           UUID : a5381c1a:fce6a693:cb201669:f728008a

  Creation Time : Sun Oct 27 16:20:43 2013

     Raid Level : raid1

  Used Dev Size : 157885888 (150.57 GiB 161.68 GB)

     Array Size : 157885888 (150.57 GiB 161.68 GB)

   Raid Devices : 2

  Total Devices : 1

Preferred Minor : 126

    Update Time : Fri Jan 16 18:30:30 2015

          State : clean

 Active Devices : 1

Working Devices : 1

 Failed Devices : 1

  Spare Devices : 0

       Checksum : bea2bc06 - correct

         Events : 4225340

      Number   Major   Minor   RaidDevice State

this     0       8       19        0      active sync   /dev/sdb3

   0     0       8       19        0      active sync   /dev/sdb3

   1     1       0        0        1      faulty removed

/dev/sdb1:

          Magic : a92b4efc

        Version : 0.90.00

           UUID : 52bda601:c015d21a:cb201669:f728008a

  Creation Time : Sun Oct 27 16:20:19 2013

     Raid Level : raid1

  Used Dev Size : 102336 (99.95 MiB 104.79 MB)

     Array Size : 102336 (99.95 MiB 104.79 MB)

   Raid Devices : 2

  Total Devices : 2

Preferred Minor : 127

    Update Time : Fri Jan 16 18:25:29 2015

          State : clean

 Active Devices : 2

Working Devices : 2

 Failed Devices : 0

  Spare Devices : 0

       Checksum : 256eeb12 - correct

         Events : 411

      Number   Major   Minor   RaidDevice State

this     0       8       17        0      active sync   /dev/sdb1

   0     0       8       17        0      active sync   /dev/sdb1

   1     1       8       33        1      active sync   /dev/sdc1

mdadm: No md superblock detected on /dev/sdb2.

/dev/sdb3:

          Magic : a92b4efc

        Version : 0.90.00

           UUID : a5381c1a:fce6a693:cb201669:f728008a

  Creation Time : Sun Oct 27 16:20:43 2013

     Raid Level : raid1

  Used Dev Size : 157885888 (150.57 GiB 161.68 GB)

     Array Size : 157885888 (150.57 GiB 161.68 GB)

   Raid Devices : 2

  Total Devices : 1

Preferred Minor : 126

    Update Time : Fri Jan 16 18:30:30 2015

          State : clean

 Active Devices : 1

Working Devices : 1

 Failed Devices : 1

  Spare Devices : 0

       Checksum : bea2bc06 - correct

         Events : 4225340

      Number   Major   Minor   RaidDevice State

this     0       8       19        0      active sync   /dev/sdb3

   0     0       8       19        0      active sync   /dev/sdb3

   1     1       0        0        1      faulty removed

/dev/sdc:

   MBR Magic : aa55

Partition[0] :       204800 sectors at         2048 (type fd)

Partition[1] :      4194304 sectors at       206848 (type 82)

Partition[2] :    315771904 sectors at      4401152 (type fd)

/dev/sdc1:

          Magic : a92b4efc

        Version : 0.90.00

           UUID : 52bda601:c015d21a:cb201669:f728008a

  Creation Time : Sun Oct 27 16:20:19 2013

     Raid Level : raid1

  Used Dev Size : 102336 (99.95 MiB 104.79 MB)

     Array Size : 102336 (99.95 MiB 104.79 MB)

   Raid Devices : 2

  Total Devices : 2

Preferred Minor : 127

    Update Time : Fri Jan 16 18:25:29 2015

          State : clean

 Active Devices : 2

Working Devices : 2

 Failed Devices : 0

  Spare Devices : 0

       Checksum : 256eeb24 - correct

         Events : 411

      Number   Major   Minor   RaidDevice State

this     1       8       33        1      active sync   /dev/sdc1

   0     0       8       17        0      active sync   /dev/sdb1

   1     1       8       33        1      active sync   /dev/sdc1

mdadm: No md superblock detected on /dev/sdc2.

mdadm: No md superblock detected on /dev/sdc3.

/dev/sdd:

   MBR Magic : aa55

Partition[0] :   3907027120 sectors at         2048 (type 83)

mdadm: No md superblock detected on /dev/sdd1.

mdadm: No md superblock detected on /dev/sde.

/dev/sdf:

   MBR Magic : aa55

Partition[0] :   4294965247 sectors at         2048 (type fd)

mdadm: No md superblock detected on /dev/sdf1.

/dev/sdg:

   MBR Magic : aa55

Partition[0] :   4294965247 sectors at         2048 (type fd)

mdadm: No md superblock detected on /dev/sdg1.

sargon ~ # mdadm /dev/md0 --stop

mdadm: No action given for /dev/md0 in --misc mode

       Action options must come before device names

sargon ~ # mdadm --stop /dev/md0

mdadm: stopped /dev/md0

sargon ~ # mdadm /dev/md126 -a /dev/sdc3

mdadm: added /dev/sdc3

sargon ~ # cat /proc/mdstat             

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdc3[2] sdb3[0]

      157885888 blocks [2/1] [U_]

      [>....................]  recovery =  0.1% (165952/157885888) finish=126.7min speed=20744K/sec

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdc3[2] sdb3[0]

      157885888 blocks [2/1] [U_]

      [>....................]  recovery =  0.4% (749760/157885888) finish=73.3min speed=35702K/sec

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdc3[2] sdb3[0]

      157885888 blocks [2/1] [U_]

      [=>...................]  recovery =  7.7% (12174016/157885888) finish=62.0min speed=39165K/sec

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdc3[2] sdb3[0]

      157885888 blocks [2/1] [U_]

      [=>...................]  recovery =  9.2% (14652032/157885888) finish=54.6min speed=43678K/sec

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdc3[2] sdb3[0]

      157885888 blocks [2/1] [U_]

      [==>..................]  recovery = 13.2% (20873152/157885888) finish=56.8min speed=40162K/sec

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdc3[2](S) sdb3[0]

      157885888 blocks [2/1] [U_]

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

unused devices: <none>

sargon ~ # 

```

Whenever I try to add the device to the raid1 array, I invariably end up with a spare that is not working.

How do I get out of this situation?

Thanks

----------

## frostschutz

what does it look like in dmesg?

if the sync is interrupted, there's usually a read or write error of some kind

----------

## NeddySeagoon

sinanqapudan,

Is the drive actually faulty?

What does dmesg say?

What about 

```
smartctl -a /dev/sdc
```

You need smartmontools for that.

----------

## sinanqapudan

frostschutz, NeddySeagoon,

this is the output of dmesg:

```
[  415.420726] md: md0 stopped.

[  605.587107] md: bind<sdc3>

[  605.618181] RAID1 conf printout:

[  605.618185]  --- wd:1 rd:2

[  605.618187]  disk 0, wo:0, o:1, dev:sdb3

[  605.618189]  disk 1, wo:1, o:1, dev:sdc3

[  605.619398] md: recovery of RAID array md126

[  605.619401] md: minimum _guaranteed_  speed: 1000 KB/sec/disk.

[  605.619403] md: using maximum available idle IO bandwidth (but not more than 200000 KB/sec) for recovery.

[  605.619405] md: using 128k window, over a total of 157885888k.

[  608.900081] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[  608.900086] ata5.00: BMDMA stat 0x64

[  608.900090] ata5.00: failed command: READ DMA EXT

[  608.900097] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[  608.900100] ata5.00: status: { DRDY ERR }

[  608.900101] ata5.00: error: { UNC }

[  608.984778] ata5.00: configured for UDMA/100

[  609.007771] ata5.01: configured for UDMA/100

[  609.007789] sd 4:0:0:0: [sdb] Unhandled sense code

[  609.007792] sd 4:0:0:0: [sdb]  

[  609.007794] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[  609.007797] sd 4:0:0:0: [sdb]  

[  609.007799] Sense Key : Medium Error [current] [descriptor]

[  609.007803] Descriptor sense data with sense descriptors (in hex):

[  609.007804]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[  609.007814]         13 15 6f f0 

[  609.007819] sd 4:0:0:0: [sdb]  

[  609.007823] Add. Sense: Unrecovered read error - auto reallocate failed

[  609.007826] sd 4:0:0:0: [sdb] CDB: 

[  609.007827] Read(10): 28 00 13 15 6f f0 00 00 08 00

[  609.007837] end_request: I/O error, dev sdb, sector 320172016

[  609.007842] Buffer I/O error on device md126, logical block 39471358

[  609.007863] ata5: EH complete

[ 1308.929831] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[ 1308.929835] ata5.00: BMDMA stat 0x64

[ 1308.929839] ata5.00: failed command: READ DMA EXT

[ 1308.929846] ata5.00: cmd 25/00:00:00:ed:5f/00:04:03:00:00/e0 tag 0 dma 524288 in

         res 51/40:00:54:ef:5f/40:04:03:00:00/e0 Emask 0x9 (media error)

[ 1308.929849] ata5.00: status: { DRDY ERR }

[ 1308.929851] ata5.00: error: { UNC }

[ 1312.756787] ata5.00: configured for UDMA/100

[ 1312.779771] ata5.01: configured for UDMA/100

[ 1312.779799] sd 4:0:0:0: [sdb] Unhandled sense code

[ 1312.779803] sd 4:0:0:0: [sdb]  

[ 1312.779805] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[ 1312.779807] sd 4:0:0:0: [sdb]  

[ 1312.779809] Sense Key : Medium Error [current] [descriptor]

[ 1312.779813] Descriptor sense data with sense descriptors (in hex):

[ 1312.779814]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[ 1312.779824]         03 5f ef 54 

[ 1312.779829] sd 4:0:0:0: [sdb]  

[ 1312.779835] Add. Sense: Unrecovered read error - auto reallocate failed

[ 1312.779837] sd 4:0:0:0: [sdb] CDB: 

[ 1312.779839] Read(10): 28 00 03 5f ed 00 00 04 00 00

[ 1312.779848] end_request: I/O error, dev sdb, sector 56618836

[ 1312.779861] ata5: EH complete

[ 1314.426719] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[ 1314.426724] ata5.00: BMDMA stat 0x64

[ 1314.426728] ata5.00: failed command: READ DMA

[ 1314.426735] ata5.00: cmd c8/00:08:50:ef:5f/00:00:00:00:00/e3 tag 0 dma 4096 in

         res 51/40:08:54:ef:5f/40:04:03:00:00/e3 Emask 0x9 (media error)

[ 1314.426738] ata5.00: status: { DRDY ERR }

[ 1314.426740] ata5.00: error: { UNC }

[ 1315.840789] ata5.00: configured for UDMA/100

[ 1315.847764] ata5.01: configured for UDMA/100

[ 1315.847784] sd 4:0:0:0: [sdb] Unhandled sense code

[ 1315.847787] sd 4:0:0:0: [sdb]  

[ 1315.847789] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[ 1315.847791] sd 4:0:0:0: [sdb]  

[ 1315.847792] Sense Key : Medium Error [current] [descriptor]

[ 1315.847796] Descriptor sense data with sense descriptors (in hex):

[ 1315.847798]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[ 1315.847808]         03 5f ef 54 

[ 1315.847813] sd 4:0:0:0: [sdb]  

[ 1315.847817] Add. Sense: Unrecovered read error - auto reallocate failed

[ 1315.847819] sd 4:0:0:0: [sdb] CDB: 

[ 1315.847820] Read(10): 28 00 03 5f ef 50 00 00 08 00

[ 1315.847830] end_request: I/O error, dev sdb, sector 56618836

[ 1315.847841] ata5: EH complete

[ 1315.847858] md/raid1:md126: sdb: unrecoverable I/O read error for block 52217600

[ 1315.847897] md: md126: recovery interrupted.

[ 1316.209076] RAID1 conf printout:

[ 1316.209079]  --- wd:1 rd:2

[ 1316.209082]  disk 0, wo:0, o:1, dev:sdb3

[ 1316.209084]  disk 1, wo:1, o:1, dev:sdc3

[ 1316.209085] RAID1 conf printout:

[ 1316.209086]  --- wd:1 rd:2

[ 1316.209088]  disk 0, wo:0, o:1, dev:sdb3

[17037.726560] spurious 8259A interrupt: IRQ7.

[17418.397774] type=1006 audit(1421446506.914:6): pid=3449 uid=0 old auid=4294967295 new auid=0 old ses=4294967295 new ses=5 res=1

sargon ~ # 

```

It seems to me that the faulty one is /dev/sdb.

Later on I will post the output of 

```
smartctl -a /dev/sdc
```

Thanks

----------

## NeddySeagoon

sinanqapudan,

```
mdadm /dev/md126 -a /dev/sdc3
```

 you were adding sdc3 back but sdb3 has read errors.

Thats really bad news.

sdc3 is now inconsistant beyond recovery as the faild sync updated some of it and sdb3 has read errors.

On the face of it time to resort to your backups.

If you don't have current validated backups, don't do anything just yet.  Post back.

----------

## sinanqapudan

Neddyseagoon,

Nere is the output of smartctl:

```
sargon ~ # smartctl -a /dev/sdc

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===

Model Family:     Maxtor DiamondMax 10 (ATA/133 and SATA/150)

Device Model:     Maxtor 6B160P0

Serial Number:    B40WJVDH

Firmware Version: BAH41B70

User Capacity:    163,928,604,672 bytes [163 GB]

Sector Size:      512 bytes logical/physical

Device is:        In smartctl database [for details use: -P show]

ATA Version is:   ATA/ATAPI-7 T13/1532D revision 0

Local Time is:    Fri Jan 16 23:54:32 2015 CET

SMART support is: Available - device has SMART capability.

SMART support is: Disabled

SMART Disabled. Use option -s with argument 'on' to enable it.

(override with '-T permissive' option)

sargon ~ # smartctl -a /dev/sdb

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===

Model Family:     Maxtor DiamondMax 10 (ATA/133 and SATA/150)

Device Model:     Maxtor 6B160P0

Serial Number:    B40SYZVH

Firmware Version: BAH41B70

User Capacity:    163,928,604,672 bytes [163 GB]

Sector Size:      512 bytes logical/physical

Device is:        In smartctl database [for details use: -P show]

ATA Version is:   ATA/ATAPI-7 T13/1532D revision 0

Local Time is:    Fri Jan 16 23:55:11 2015 CET

SMART support is: Available - device has SMART capability.

SMART support is: Disabled

SMART Disabled. Use option -s with argument 'on' to enable it.

(override with '-T permissive' option)

sargon ~ # 
```

As far as backups are concerned, I made one before attempting to add the spare by issuing:

```
# rsync -avxHAS --delete --progress
```

But I don't know if that counts as validated.

----------

## NeddySeagoon

sinanqapudan,

```
SMART support is: Disabled 
```

Enable SMART support for both sdb and sdc and post the output of both

```
smartctl -a /dev/sdb

smartctl -a /dev/sdc
```

----------

## sinanqapudan

NeddySeagoon,

Here's the output of smartctl. In effect it appears that there are errore on sdb.

```
sargon ~ # smartctl -s on /dev/sdb

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF ENABLE/DISABLE COMMANDS SECTION ===

SMART Enabled.

sargon ~ # smartctl -s on /dev/sdc

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF ENABLE/DISABLE COMMANDS SECTION ===

SMART Enabled.

sargon ~ # smartctl -a  /dev/sdc

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===

Model Family:     Maxtor DiamondMax 10 (ATA/133 and SATA/150)

Device Model:     Maxtor 6B160P0

Serial Number:    B40WJVDH

Firmware Version: BAH41B70

User Capacity:    163,928,604,672 bytes [163 GB]

Sector Size:      512 bytes logical/physical

Device is:        In smartctl database [for details use: -P show]

ATA Version is:   ATA/ATAPI-7 T13/1532D revision 0

Local Time is:    Sat Jan 17 09:34:22 2015 CET

SMART support is: Available - device has SMART capability.

SMART support is: Enabled

=== START OF READ SMART DATA SECTION ===

SMART overall-health self-assessment test result: PASSED

General SMART Values:

Offline data collection status:  (0x85) Offline data collection activity                                        was aborted by an interrupting command from host.

                                        Auto Offline Data Collection: Enabled.

Self-test execution status:      (   0) The previous self-test routine completed

                                        without error or no self-test has ever

                                        been run.

Total time to complete Offline

data collection:                ( 1622) seconds.

Offline data collection

capabilities:                    (0x5b) SMART execute Offline immediate.                                        Auto Offline data collection on/off support.

                                        Suspend Offline collection upon new

                                        command.

                                        Offline surface scan supported.

                                        Self-test supported.

                                        No Conveyance Self-test supported.

                                        Selective Self-test supported.

SMART capabilities:            (0x0003) Saves SMART data before entering                                        power-saving mode.

                                        Supports SMART auto save timer.

Error logging capability:        (0x01) Error logging supported.

                                        No General Purpose Logging support.

Short self-test routine

recommended polling time:        (   2) minutes.

Extended self-test routine

recommended polling time:        (  72) minutes.

SMART Attributes Data Structure revision number: 16

Vendor Specific SMART Attributes with Thresholds:

ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE

  3 Spin_Up_Time            0x0027   205   204   063    Pre-fail  Always       -       14630

  4 Start_Stop_Count        0x0032   252   252   000    Old_age   Always       -       2724

  5 Reallocated_Sector_Ct   0x0033   253   253   063    Pre-fail  Always       -       0

  6 Read_Channel_Margin     0x0001   253   253   100    Pre-fail  Offline      -       0

  7 Seek_Error_Rate         0x000a   253   252   000    Old_age   Always       -       0

  8 Seek_Time_Performance   0x0027   250   241   187    Pre-fail  Always       -       36416

  9 Power_On_Minutes        0x0032   174   174   000    Old_age   Always       -       145h+08m

 10 Spin_Retry_Count        0x002b   253   252   157    Pre-fail  Always       -       0

 11 Calibration_Retry_Count 0x002b   253   252   223    Pre-fail  Always       -       0

 12 Power_Cycle_Count       0x0032   244   244   000    Old_age   Always       -       3650

192 Power-Off_Retract_Count 0x0032   253   253   000    Old_age   Always       -       0

193 Load_Cycle_Count        0x0032   253   253   000    Old_age   Always       -       0

194 Temperature_Celsius     0x0032   044   253   000    Old_age   Always       -       48

195 Hardware_ECC_Recovered  0x000a   253   252   000    Old_age   Always       -       4551

196 Reallocated_Event_Count 0x0008   250   250   000    Old_age   Offline      -       3

197 Current_Pending_Sector  0x0008   253   253   000    Old_age   Offline      -       0

198 Offline_Uncorrectable   0x0008   249   249   000    Old_age   Offline      -       4

199 UDMA_CRC_Error_Count    0x0008   199   199   000    Old_age   Offline      -       0

200 Multi_Zone_Error_Rate   0x000a   253   252   000    Old_age   Always       -       0

201 Soft_Read_Error_Rate    0x000a   253   252   000    Old_age   Always       -       0

202 Data_Address_Mark_Errs  0x000a   253   252   000    Old_age   Always       -       0

203 Run_Out_Cancel          0x000b   253   252   180    Pre-fail  Always       -       0

204 Soft_ECC_Correction     0x000a   253   252   000    Old_age   Always       -       0

205 Thermal_Asperity_Rate   0x000a   253   252   000    Old_age   Always       -       0

207 Spin_High_Current       0x002a   253   252   000    Old_age   Always       -       0

208 Spin_Buzz               0x002a   253   252   000    Old_age   Always       -       0

209 Offline_Seek_Performnce 0x0024   240   240   000    Old_age   Offline      -       164

210 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

211 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

212 Unknown_Attribute       0x0032   253   253   000    Old_age   Always       -       0

SMART Error Log Version: 1

No Errors Logged

SMART Self-test log structure revision number 1

No self-tests have been logged.  [To run self-tests, use: smartctl -t]

SMART Selective self-test log data structure revision number 1

 SPAN  MIN_LBA  MAX_LBA  CURRENT_TEST_STATUS

    1        0        0  Not_testing

    2        0        0  Not_testing

    3        0        0  Not_testing

    4        0        0  Not_testing

    5        0        0  Not_testing

Selective self-test flags (0x0):

  After scanning selected spans, do NOT read-scan remainder of disk.

If Selective self-test is pending on power-up, resume after 0 minute delay.

sargon ~ # smartctl -a  /dev/sdb

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===

Model Family:     Maxtor DiamondMax 10 (ATA/133 and SATA/150)

Device Model:     Maxtor 6B160P0

Serial Number:    B40SYZVH

Firmware Version: BAH41B70

User Capacity:    163,928,604,672 bytes [163 GB]

Sector Size:      512 bytes logical/physical

Device is:        In smartctl database [for details use: -P show]

ATA Version is:   ATA/ATAPI-7 T13/1532D revision 0

Local Time is:    Sat Jan 17 09:34:29 2015 CET

SMART support is: Available - device has SMART capability.

SMART support is: Enabled

=== START OF READ SMART DATA SECTION ===

SMART overall-health self-assessment test result: PASSED

General SMART Values:

Offline data collection status:  (0x82) Offline data collection activity                                        was completed without error.

                                        Auto Offline Data Collection: Enabled.

Self-test execution status:      (   0) The previous self-test routine completed

                                        without error or no self-test has ever

                                        been run.

Total time to complete Offline

data collection:                ( 1622) seconds.

Offline data collection

capabilities:                    (0x5b) SMART execute Offline immediate.                                        Auto Offline data collection on/off support.

                                        Suspend Offline collection upon new

                                        command.

                                        Offline surface scan supported.

                                        Self-test supported.

                                        No Conveyance Self-test supported.

                                        Selective Self-test supported.

SMART capabilities:            (0x0003) Saves SMART data before entering                                        power-saving mode.

                                        Supports SMART auto save timer.

Error logging capability:        (0x01) Error logging supported.

                                        No General Purpose Logging support.

Short self-test routine

recommended polling time:        (   2) minutes.

Extended self-test routine

recommended polling time:        (  72) minutes.

SMART Attributes Data Structure revision number: 16

Vendor Specific SMART Attributes with Thresholds:

ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE

  3 Spin_Up_Time            0x0027   213   203   063    Pre-fail  Always       -       12041

  4 Start_Stop_Count        0x0032   253   253   000    Old_age   Always       -       1823

  5 Reallocated_Sector_Ct   0x0033   252   240   063    Pre-fail  Always       -       19

  6 Read_Channel_Margin     0x0001   253   253   100    Pre-fail  Offline      -       0

  7 Seek_Error_Rate         0x000a   253   252   000    Old_age   Always       -       0

  8 Seek_Time_Performance   0x0027   250   239   187    Pre-fail  Always       -       37878

  9 Power_On_Minutes        0x0032   177   177   000    Old_age   Always       -       334h+03m

 10 Spin_Retry_Count        0x002b   253   252   157    Pre-fail  Always       -       0

 11 Calibration_Retry_Count 0x002b   253   252   223    Pre-fail  Always       -       0

 12 Power_Cycle_Count       0x0032   247   247   000    Old_age   Always       -       2549

192 Power-Off_Retract_Count 0x0032   253   253   000    Old_age   Always       -       0

193 Load_Cycle_Count        0x0032   253   253   000    Old_age   Always       -       0

194 Temperature_Celsius     0x0032   044   253   000    Old_age   Always       -       48

195 Hardware_ECC_Recovered  0x000a   253   252   000    Old_age   Always       -       8920

196 Reallocated_Event_Count 0x0008   124   124   000    Old_age   Offline      -       129

197 Current_Pending_Sector  0x0008   253   241   000    Old_age   Offline      -       4

198 Offline_Uncorrectable   0x0008   124   124   000    Old_age   Offline      -       129

199 UDMA_CRC_Error_Count    0x0008   199   199   000    Old_age   Offline      -       0

200 Multi_Zone_Error_Rate   0x000a   253   252   000    Old_age   Always       -       0

201 Soft_Read_Error_Rate    0x000a   253   252   000    Old_age   Always       -       0

202 Data_Address_Mark_Errs  0x000a   253   247   000    Old_age   Always       -       0

203 Run_Out_Cancel          0x000b   253   252   180    Pre-fail  Always       -       0

204 Soft_ECC_Correction     0x000a   253   252   000    Old_age   Always       -       0

205 Thermal_Asperity_Rate   0x000a   253   252   000    Old_age   Always       -       0

207 Spin_High_Current       0x002a   253   252   000    Old_age   Always       -       0

208 Spin_Buzz               0x002a   253   252   000    Old_age   Always       -       0

209 Offline_Seek_Performnce 0x0024   239   239   000    Old_age   Offline      -       168

210 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

211 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

212 Unknown_Attribute       0x0032   253   253   000    Old_age   Always       -       0

SMART Error Log Version: 1

ATA Error Count: 740 (device log contains only the most recent five errors)

        CR = Command Register [HEX]

        FR = Features Register [HEX]

        SC = Sector Count Register [HEX]

        SN = Sector Number Register [HEX]

        CL = Cylinder Low Register [HEX]

        CH = Cylinder High Register [HEX]

        DH = Device/Head Register [HEX]

        DC = Device Command Register [HEX]

        ER = Error register [HEX]

        ST = Status register [HEX]

Powered_Up_Time is measured from power on, and printed as

DDd+hh:mm:SS.sss where DD=days, hh=hours, mm=minutes,

SS=sec, and sss=millisec. It "wraps" after 49.710 days.

Error 740 occurred at disk power-on lifetime: 16060 hours (669 days + 4 hours)

  When the command that caused the error occurred, the device was in an unknown state.

  After command completion occurred, registers were:

  ER ST SC SN CL CH DH

  -- -- -- -- -- -- --

  5a 4a 00 00 00 00 a0

  Commands leading to the command that caused the error were:

  CR FR SC SN CL CH DH DC   Powered_Up_Time  Command/Feature_Name

  -- -- -- -- -- -- -- --  ----------------  --------------------

  a0 00 00 00 00 00 a0 00      00:23:14.618  PACKET

  a0 00 08 50 ef 5f e3 00      00:23:13.224  PACKET

  a0 00 08 48 ef 5f e3 00      00:23:13.224  PACKET

  a0 00 08 40 ef 5f e3 00      00:23:13.223  PACKET

  a0 00 08 38 ef 5f e3 00      00:23:13.223  PACKET

Error 739 occurred at disk power-on lifetime: 16060 hours (669 days + 4 hours)

  When the command that caused the error occurred, the device was in an unknown state.

  After command completion occurred, registers were:

  ER ST SC SN CL CH DH

  -- -- -- -- -- -- --

  5a 4a 00 00 00 00 a0

  Commands leading to the command that caused the error were:

  CR FR SC SN CL CH DH DC   Powered_Up_Time  Command/Feature_Name

  -- -- -- -- -- -- -- --  ----------------  --------------------

  a0 00 00 00 00 00 a0 00      00:23:12.850  PACKET

  a0 00 00 00 ed 5f e0 00      00:23:05.414  PACKET

  a0 00 00 00 e9 5f e0 00      00:23:05.402  PACKET

  a0 00 80 80 e5 5f e0 00      00:23:05.391  PACKET

  a0 00 00 80 e1 5f e0 00      00:23:05.379  PACKET

Error 738 occurred at disk power-on lifetime: 16060 hours (669 days + 4 hours)

  When the command that caused the error occurred, the device was in an unknown state.

  After command completion occurred, registers were:

  ER ST SC SN CL CH DH

  -- -- -- -- -- -- --

  5a 4a 00 00 00 00 a0

  Commands leading to the command that caused the error were:

  CR FR SC SN CL CH DH DC   Powered_Up_Time  Command/Feature_Name

  -- -- -- -- -- -- -- --  ----------------  --------------------

  a0 00 00 00 00 00 a0 00      00:11:29.115  PACKET

  a0 00 08 f0 6f 15 e0 00      00:11:25.997  PACKET

  a0 00 08 80 73 15 e0 00      00:11:25.983  PACKET

  a0 00 00 00 47 43 e0 00      00:11:25.971  PACKET

  a0 00 80 80 43 43 e0 00      00:11:25.936  PACKET

Error 737 occurred at disk power-on lifetime: 16060 hours (669 days + 4 hours)

  When the command that caused the error occurred, the device was in an unknown state.

  After command completion occurred, registers were:

  ER ST SC SN CL CH DH

  -- -- -- -- -- -- --

  5a 4a 00 00 00 00 a0

  Commands leading to the command that caused the error were:

  CR FR SC SN CL CH DH DC   Powered_Up_Time  Command/Feature_Name

  -- -- -- -- -- -- -- --  ----------------  --------------------

  a0 00 00 00 00 00 a0 00      00:02:00.512  PACKET

  a0 00 08 f0 6f 15 e0 00      00:01:57.420  PACKET

  a0 00 08 80 73 15 e0 00      00:01:57.418  PACKET

  a0 00 00 07 28 43 a0 00      00:01:57.382  PACKET

  a0 00 08 00 28 43 e0 00      00:01:57.382  PACKET

Error 736 occurred at disk power-on lifetime: 16060 hours (669 days + 4 hours)

  When the command that caused the error occurred, the device was in an unknown state.

  After command completion occurred, registers were:

  ER ST SC SN CL CH DH

  -- -- -- -- -- -- --

  5a 4a 00 00 00 00 a0

  Commands leading to the command that caused the error were:

  CR FR SC SN CL CH DH DC   Powered_Up_Time  Command/Feature_Name

  -- -- -- -- -- -- -- --  ----------------  --------------------

  a0 00 00 00 00 00 a0 00      00:01:56.187  PACKET

  a0 00 08 f0 6f 15 e0 00      00:01:53.095  PACKET

  a0 00 08 78 08 00 e0 00      00:01:53.095  PACKET

  a0 00 08 38 08 00 e0 00      00:01:53.095  PACKET

  a0 00 08 18 08 00 e0 00      00:01:53.094  PACKET

SMART Self-test log structure revision number 1

No self-tests have been logged.  [To run self-tests, use: smartctl -t]

SMART Selective self-test log data structure revision number 1

 SPAN  MIN_LBA  MAX_LBA  CURRENT_TEST_STATUS

    1        0        0  Not_testing

    2        0        0  Not_testing

    3        0        0  Not_testing

    4        0        0  Not_testing

    5        0        0  Not_testing

Selective self-test flags (0x0):

  After scanning selected spans, do NOT read-scan remainder of disk.

If Selective self-test is pending on power-up, resume after 0 minute delay.

sargon ~ #
```

----------

## NeddySeagoon

sinanqapudan,

```
smartctl -s on /dev/sdc 

ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE 

  5 Reallocated_Sector_Ct   0x0033   253   253   063    Pre-fail  Always       -       0 

196 Reallocated_Event_Count 0x0008   250   250   000    Old_age   Offline      -       3

197 Current_Pending_Sector  0x0008   253   253   000    Old_age   Offline      -       0 
```

```
smartctl -a  /dev/sdb

ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE 

  5 Reallocated_Sector_Ct   0x0033   252   240   063    Pre-fail  Always       -       19 

196 Reallocated_Event_Count 0x0008   124   124   000    Old_age   Offline      -       129

197 Current_Pending_Sector  0x0008   253   241   000    Old_age   Offline      -       4 
```

These are the inportant numbers.  VALUE, WORST and THRESH are all normalised values. If  VALUE or WORST is less than or equal to THRESH, the parametor has failed.

RAW_VALUEs are often difficult to interpret as they are 32 bit nubers and several raw values can be packed into the 32 bits.  That means that high numbers are not always a cause for concern.

Taking the values above,  /dev/sdc has no known faulty sectors. Its possible that some exist but they have not been read.

/dev/sdb is different. It has 19 reallocated sectors.  Thats not a problom as there are plenty left.  New drives can have reallocated sectors and further sectors are reallocted as the drive determines that they are getting difficult to read.

The problem here is the Current_Pending_Sector  count.  The drive knows about 4 sectors that it would like to reallocate but that it can no longer read. Consider the data here as lost.

There may be more.

From your dmesg

```
[  609.007837] end_request: I/O error, dev sdb, sector 320172016

[ 1312.779848] end_request: I/O error, dev sdb, sector 56618836
```

so we know the locations of two of them.

What happens next depends on your luck.

Resyncing the raid reads the entire partitions, unallocated space included. If you are lucky, these unreadable sectors are in the unallocated space in the filesystem.

If these sectors are it use by the filesystem, the data loss depends on what they contain. Blocks of a file, blocks of a directory, filesystem metadata ... ?

Your backup 

```
rsync -avxHAS --delete --progress
```

will have bailed out at the fist error, so it may well be incomplete, so lets say you don't have a backup.

Look at the end of dmesg and note the timestamp of the last message.

Run the backup again then look at dmesg for any nasty

```
[  609.007826] sd 4:0:0:0: [sdb] CDB:

[  609.007827] Read(10): 28 00 13 15 6f f0 00 00 08 00

[  609.007837] end_request: I/O error, dev sdb, sector 320172016

[  609.007842] Buffer I/O error on device md126, logical block 39471358

[  609.007863] ata5: EH complete 
```

errors that have been added.  If there are none, rsync completed and we can conclude that the errors are in unallocated filesystem space.

If there is an error, rsync did not complete and you don't have a backup.  More drastic data recovery is called for.

Do you feel lucky?

I have not checked the warraty status of your drives but a non zero Current_Pending_Sector count means that the drive cannot read its own writing.

That's grounds for a warranty replacement, if the drive is still in warranty.

----------

## sinanqapudan

NeddySeagoon,

/deve/md126 is my root and tthe system appears to be working just fine.

Maybe as you wrote the errors are limited to unallocated sectors.

We'll see....

----------

## sinanqapudan

NeddySeagoon,

some errors have indeed cropped up in dmesg:

```
[81777.242990] type=1006 audit(1421510865.760:12): pid=11350 uid=0 old auid=4294967295 new auid=0 old ses=4294967295 new ses=11 res=1

[82788.459754] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[82788.459758] ata5.00: BMDMA stat 0x64

[82788.459763] ata5.00: failed command: READ DMA

[82788.459770] ata5.00: cmd c8/00:00:48:ef:5f/00:00:00:00:00/e3 tag 0 dma 131072 in

         res 51/40:00:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[82788.459772] ata5.00: status: { DRDY ERR }

[82788.459774] ata5.00: error: { UNC }

[82792.287777] ata5.00: configured for UDMA/100

[82792.295424] ata5.01: configured for UDMA/100

[82792.295445] sd 4:0:0:0: [sdb] Unhandled sense code

[82792.295448] sd 4:0:0:0: [sdb]  

[82792.295450] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[82792.295452] sd 4:0:0:0: [sdb]  

[82792.295454] Sense Key : Medium Error [current] [descriptor]

[82792.295459] Descriptor sense data with sense descriptors (in hex):

[82792.295461]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[82792.295471]         03 5f ef 54 

[82792.295475] sd 4:0:0:0: [sdb]  

[82792.295480] Add. Sense: Unrecovered read error - auto reallocate failed

[82792.295482] sd 4:0:0:0: [sdb] CDB: 

[82792.295483] Read(10): 28 00 03 5f ef 48 00 01 00 00

[82792.295493] end_request: I/O error, dev sdb, sector 56618836

[82792.295518] ata5: EH complete

[82793.791599] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[82793.791602] ata5.00: BMDMA stat 0x64

[82793.791607] ata5.00: failed command: READ DMA

[82793.791614] ata5.00: cmd c8/00:08:50:ef:5f/00:00:00:00:00/e3 tag 0 dma 4096 in

         res 51/40:08:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[82793.791616] ata5.00: status: { DRDY ERR }

[82793.791618] ata5.00: error: { UNC }

[82793.878783] ata5.00: configured for UDMA/100

[82793.885777] ata5.01: configured for UDMA/100

[82793.885790] sd 4:0:0:0: [sdb] Unhandled sense code

[82793.885792] sd 4:0:0:0: [sdb]  

[82793.885794] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[82793.885797] sd 4:0:0:0: [sdb]  

[82793.885798] Sense Key : Medium Error [current] [descriptor]

[82793.885802] Descriptor sense data with sense descriptors (in hex):

[82793.885804]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[82793.885814]         03 5f ef 54 

[82793.885819] sd 4:0:0:0: [sdb]  

[82793.885823] Add. Sense: Unrecovered read error - auto reallocate failed

[82793.885826] sd 4:0:0:0: [sdb] CDB: 

[82793.885827] Read(10): 28 00 03 5f ef 50 00 00 08 00

[82793.885836] end_request: I/O error, dev sdb, sector 56618836

[82793.885851] ata5: EH complete

[82797.649692] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[82797.649698] ata5.00: BMDMA stat 0x64

[82797.649702] ata5.00: failed command: READ DMA

[82797.649709] ata5.00: cmd c8/00:08:50:ef:5f/00:00:00:00:00/e3 tag 0 dma 4096 in

         res 51/40:08:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[82797.649712] ata5.00: status: { DRDY ERR }

[82797.649714] ata5.00: error: { UNC }

[82797.736774] ata5.00: configured for UDMA/100

[82797.743804] ata5.01: configured for UDMA/100

[82797.743824] sd 4:0:0:0: [sdb] Unhandled sense code

[82797.743827] sd 4:0:0:0: [sdb]  

[82797.743829] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[82797.743831] sd 4:0:0:0: [sdb]  

[82797.743832] Sense Key : Medium Error [current] [descriptor]

[82797.743836] Descriptor sense data with sense descriptors (in hex):

[82797.743838]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[82797.743848]         03 5f ef 54 

[82797.743853] sd 4:0:0:0: [sdb]  

[82797.743857] Add. Sense: Unrecovered read error - auto reallocate failed

[82797.743860] sd 4:0:0:0: [sdb] CDB: 

[82797.743861] Read(10): 28 00 03 5f ef 50 00 00 08 00

[82797.743871] end_request: I/O error, dev sdb, sector 56618836

[82797.743894] ata5: EH complete

[82823.214334] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[82823.214339] ata5.00: BMDMA stat 0x64

[82823.214343] ata5.00: failed command: READ DMA

[82823.214350] ata5.00: cmd c8/00:08:50:ef:5f/00:00:00:00:00/e3 tag 0 dma 4096 in

         res 51/40:08:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[82823.214352] ata5.00: status: { DRDY ERR }

[82823.214354] ata5.00: error: { UNC }

[82823.302782] ata5.00: configured for UDMA/100

[82823.309778] ata5.01: configured for UDMA/100

[82823.309795] sd 4:0:0:0: [sdb] Unhandled sense code

[82823.309798] sd 4:0:0:0: [sdb]  

[82823.309800] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[82823.309802] sd 4:0:0:0: [sdb]  

[82823.309804] Sense Key : Medium Error [current] [descriptor]

[82823.309809] Descriptor sense data with sense descriptors (in hex):

[82823.309810]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[82823.309820]         03 5f ef 54 

[82823.309825] sd 4:0:0:0: [sdb]  

[82823.309829] Add. Sense: Unrecovered read error - auto reallocate failed

[82823.309832] sd 4:0:0:0: [sdb] CDB: 

[82823.309833] Read(10): 28 00 03 5f ef 50 00 00 08 00

[82823.309843] end_request: I/O error, dev sdb, sector 56618836

[82823.309862] ata5: EH complete

[82824.722376] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[82824.722380] ata5.00: BMDMA stat 0x64

[82824.722386] ata5.00: failed command: READ DMA

[82824.722393] ata5.00: cmd c8/00:08:50:ef:5f/00:00:00:00:00/e3 tag 0 dma 4096 in

         res 51/40:08:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[82824.722396] ata5.00: status: { DRDY ERR }

[82824.722398] ata5.00: error: { UNC }

[82824.809783] ata5.00: configured for UDMA/100

[82824.816812] ata5.01: configured for UDMA/100

[82824.816833] sd 4:0:0:0: [sdb] Unhandled sense code

[82824.816835] sd 4:0:0:0: [sdb]  

[82824.816837] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[82824.816840] sd 4:0:0:0: [sdb]  

[82824.816841] Sense Key : Medium Error [current] [descriptor]

[82824.816845] Descriptor sense data with sense descriptors (in hex):

[82824.816847]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[82824.816857]         03 5f ef 54 

[82824.816861] sd 4:0:0:0: [sdb]  

[82824.816867] Add. Sense: Unrecovered read error - auto reallocate failed

[82824.816870] sd 4:0:0:0: [sdb] CDB: 

[82824.816871] Read(10): 28 00 03 5f ef 50 00 00 08 00

[82824.816880] end_request: I/O error, dev sdb, sector 56618836

[82824.816896] ata5: EH complete

```

which, by the way, was to be expected as rsync ended with:

```
var/tmp/portage/net-misc/r8168-8.028.00/work/r8168-8.028.00/src/rtltool.h

         723 100%    0.79kB/s    0:00:00 (xfer#392733, to-check=1/439500)

var/tmp/portage/net-misc/r8168-8.028.00/work/r8168-8.028.00/src/.tmp_versions/

WARNING: var/log/messages failed verification -- update discarded (will try again).

var/log/messages

   374104333 100%   25.71MB/s    0:00:13 (xfer#392734, to-check=466/439500)

rsync: read errors mapping "/var/log/messages": Input/output error (5)

ERROR: var/log/messages failed verification -- update discarded.

sent 6560470642 bytes  received 7655901 bytes  5023423.74 bytes/sec

total size is 6160833468  speedup is 0.94

rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1052) [sender=3.0.9]

```

Soooo, what next? Oh and I don't feel particularly lucky when it comes to  mdadm and raid...

But I would really to have both the system working (as it is now) and raid1 for high availability, even if it is a non mission critical machine...

Nothing appears to be missing from the backup, except for the file  var/log/messages

eagerly awaiting for further instructions

----------

## NeddySeagoon

sinanqapudan,

That /var/log/messages faiied is to be expected.  It was appended to during the rsync by the read errors.

The errors were all for  

```
[82797.743871] end_request: I/O error, dev sdb, sector 56618836.
```

```
rsync: read errors mapping "/var/log/messages": Input/output error (5)
```

 looks really encouraging.

As if the bad block belongs to /var/log/messages too.

We can test that in several ways. The easiest is to copy /var/log/messages somewhere.  

You should get a read error at sector 56618836.  If that happens, your backup is good enough.

Has the smart output changed any from

```
smartctl -a  /dev/sdb

ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE

  5 Reallocated_Sector_Ct   0x0033   252   240   063    Pre-fail  Always       -       19

196 Reallocated_Event_Count 0x0008   124   124   000    Old_age   Offline      -       129

197 Current_Pending_Sector  0x0008   253   241   000    Old_age   Offline      -       4 
```

----------

## sinanqapudan

NeddySeagoon,

the output of 

```
smartctl -a  /dev/sdb
```

doesn't appear to have changed a lot (to me at least). Here it is:

```
sargon sysbak1 # smartctl -a  /dev/sdb

smartctl 6.3 2014-07-26 r3976 [i686-linux-3.13.6-hardened-r3] (local build)

Copyright (C) 2002-14, Bruce Allen, Christian Franke, www.smartmontools.org

=== START OF INFORMATION SECTION ===

Model Family:     Maxtor DiamondMax 10 (ATA/133 and SATA/150)

Device Model:     Maxtor 6B160P0

Serial Number:    B40SYZVH

Firmware Version: BAH41B70

User Capacity:    163,928,604,672 bytes [163 GB]

Sector Size:      512 bytes logical/physical

Device is:        In smartctl database [for details use: -P show]

ATA Version is:   ATA/ATAPI-7 T13/1532D revision 0

Local Time is:    Sat Jan 17 21:39:34 2015 CET

SMART support is: Available - device has SMART capability.

SMART support is: Enabled

=== START OF READ SMART DATA SECTION ===

SMART overall-health self-assessment test result: PASSED

General SMART Values:

Offline data collection status:  (0x82) Offline data collection activity

                                        was completed without error.

                                        Auto Offline Data Collection: Enabled.

Self-test execution status:      (   0) The previous self-test routine completed

                                        without error or no self-test has ever 

                                        been run.

Total time to complete Offline 

data collection:                ( 1622) seconds.

Offline data collection

capabilities:                    (0x5b) SMART execute Offline immediate.

                                        Auto Offline data collection on/off support.

                                        Suspend Offline collection upon new

                                        command.

                                        Offline surface scan supported.

                                        Self-test supported.

                                        No Conveyance Self-test supported.

                                        Selective Self-test supported.

SMART capabilities:            (0x0003) Saves SMART data before entering

                                        power-saving mode.

                                        Supports SMART auto save timer.

Error logging capability:        (0x01) Error logging supported.

                                        No General Purpose Logging support.

Short self-test routine 

recommended polling time:        (   2) minutes.

Extended self-test routine

recommended polling time:        (  72) minutes.

SMART Attributes Data Structure revision number: 16

Vendor Specific SMART Attributes with Thresholds:

ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE

  3 Spin_Up_Time            0x0027   213   203   063    Pre-fail  Always       -       12041

  4 Start_Stop_Count        0x0032   253   253   000    Old_age   Always       -       1823

  5 Reallocated_Sector_Ct   0x0033   252   240   063    Pre-fail  Always       -       19

  6 Read_Channel_Margin     0x0001   253   253   100    Pre-fail  Offline      -       0

  7 Seek_Error_Rate         0x000a   253   252   000    Old_age   Always       -       0

  8 Seek_Time_Performance   0x0027   246   239   187    Pre-fail  Always       -       33909

  9 Power_On_Minutes        0x0032   177   177   000    Old_age   Always       -       346h+04m

 10 Spin_Retry_Count        0x002b   253   252   157    Pre-fail  Always       -       0

 11 Calibration_Retry_Count 0x002b   253   252   223    Pre-fail  Always       -       0

 12 Power_Cycle_Count       0x0032   247   247   000    Old_age   Always       -       2549

192 Power-Off_Retract_Count 0x0032   253   253   000    Old_age   Always       -       0

193 Load_Cycle_Count        0x0032   253   253   000    Old_age   Always       -       0

194 Temperature_Celsius     0x0032   044   253   000    Old_age   Always       -       47

195 Hardware_ECC_Recovered  0x000a   253   252   000    Old_age   Always       -       10980

196 Reallocated_Event_Count 0x0008   124   124   000    Old_age   Offline      -       129

197 Current_Pending_Sector  0x0008   253   241   000    Old_age   Offline      -       4

198 Offline_Uncorrectable   0x0008   124   124   000    Old_age   Offline      -       129

199 UDMA_CRC_Error_Count    0x0008   199   199   000    Old_age   Offline      -       0

200 Multi_Zone_Error_Rate   0x000a   253   252   000    Old_age   Always       -       0

201 Soft_Read_Error_Rate    0x000a   253   252   000    Old_age   Always       -       0

202 Data_Address_Mark_Errs  0x000a   253   247   000    Old_age   Always       -       0

203 Run_Out_Cancel          0x000b   253   252   180    Pre-fail  Always       -       2

204 Soft_ECC_Correction     0x000a   253   252   000    Old_age   Always       -       0

205 Thermal_Asperity_Rate   0x000a   253   252   000    Old_age   Always       -       0

207 Spin_High_Current       0x002a   253   252   000    Old_age   Always       -       0

208 Spin_Buzz               0x002a   253   252   000    Old_age   Always       -       0

209 Offline_Seek_Performnce 0x0024   239   239   000    Old_age   Offline      -       168

210 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

211 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

212 Unknown_Attribute       0x0032   253   253   000    Old_age   Always       -       0

SMART Error Log Version: 1

ATA Error Count: 746 (device log contains only the most recent five errors)

        CR = Command Register [HEX]

        FR = Features Register [HEX]

        SC = Sector Count Register [HEX]

        SN = Sector Number Register [HEX]

        CL = Cylinder Low Register [HEX]

        CH = Cylinder High Register [HEX]

        DH = Device/Head Register [HEX]

        DC = Device Command Register [HEX]

        ER = Error register [HEX]

        ST = Status register [HEX]

Powered_Up_Time is measured from power on, and printed as

DDd+hh:mm:SS.sss where DD=days, hh=hours, mm=minutes,

SS=sec, and sss=millisec. It "wraps" after 49.710 days.

Error 746 occurred at disk power-on lifetime: 24896 hours (1037 days + 8 hours)

  When the command that caused the error occurred, the device was in an unknown state.

  After command completion occurred, registers were:

  ER ST SC SN CL CH DH

  -- -- -- -- -- -- --

  5a 4a 00 00 00 00 a0

  Commands leading to the command that caused the error were:

  CR FR SC SN CL CH DH DC   Powered_Up_Time  Command/Feature_Name

  -- -- -- -- -- -- -- --  ----------------  --------------------

  a0 00 00 00 00 00 a0 00      23:01:06.884  PACKET

  a0 00 00 48 ef 5f e3 00      23:00:59.421  PACKET

  a0 00 00 48 ee 5f e3 00      23:00:59.420  PACKET

  a0 00 00 48 ed 5f e3 00      23:00:59.413  PACKET

  a0 00 00 48 ec 5f e3 00      23:00:59.410  PACKET

SMART Self-test log structure revision number 1

No self-tests have been logged.  [To run self-tests, use: smartctl -t]

SMART Selective self-test log data structure revision number 1

 SPAN  MIN_LBA  MAX_LBA  CURRENT_TEST_STATUS

    1        0        0  Not_testing

    2        0        0  Not_testing

    3        0        0  Not_testing

    4        0        0  Not_testing

    5        0        0  Not_testing

Selective self-test flags (0x0):

  After scanning selected spans, do NOT read-scan remainder of disk.

If Selective self-test is pending on power-up, resume after 0 minute delay.

```

The copy of /var/log/messages fails due to exactly what you predicted:

```
sargon / # cp -v /var/log/messages /mt/sysbak1/sdb3

‘/var/log/messages’ -> ‘/mt/sysbak1/sdb3/messages’

cp: error reading ‘/var/log/messages’: Input/output error

cp: failed to extend ‘/mt/sysbak1/sdb3/messages’: Input/output error
```

and here it is the error in the dmesg:

```
[98405.772845] end_request: I/O error, dev sdb, sector 56618836
```

Regards

----------

## NeddySeagoon

sinanqapudan,

Your luck is in!

The disk has not got any worse for continued use and 

The copy gave

```
[98405.772845] end_request: I/O error, dev sdb, sector 56618836
```

and rsync gave 

```
[82797.743871] end_request: I/O error, dev sdb, sector 56618836
```

So we know that the filesystem has a single unreadable block at 56618836 and its in /var/log/messages, which we can manage without.

Proceede as follows.

Make a whole new degraded raid set on /dev/sdc3.  It needs to be made with a missing drive as we will add sdb3, or its replacement, to it later.

Restore your backup to this new degraded raid set.

You now have /dev/md126 in degraded mode with only /dev/sdb3 and a new degraded mode raid with only /dev/sdc3 in it.

They are almost identical - we need to be able to tell them apart.

Add a file  to the top level of the new raid. 

```
touch /mnt/gentoo/new_raid
```

will do nicely.

Edit /ect/fstab on the new raid to update the /dev/md126 entry.

Add a new booting stanza to your boot loader so you have the option to use either  /dev/md126 as root (thats already there) or the new raid.

Fix your raid set assembly process to assemble the new raid as well an the old one - I don't know how you do that but I guess that its an initrd update.

Boot into your restored backup degraded raid set. Check that the file /new_raid is present.  Check things out.

You are now running on your new degraded raid set made from your backup.

Think about drive replacement before you continue.

When you are happy, stop /dev/md126, fail and remove /dev/sdb3 and destroy the raid metadata on /dev/sdb3

Now add /dev/sdb3 to the new raid set to bring it up to strength.

Syncing from /dev/sdc3 to /dev/sdb3 will overwrite the entire partition.  The writes will force relocation of the pending sectors. They might not all be in partition 3.

When the sync completes, you are back in business.

Fix your boot loader not to offer to boot from /dev/md126 and fix your initrd not to attempt to assemble /dev/md126.

Check 	smartctl -a  /dev/sdb

```
ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE

  5 Reallocated_Sector_Ct   0x0033   252   240   063    Pre-fail  Always       -       19

196 Reallocated_Event_Count 0x0008   124   124   000    Old_age   Offline      -       129

197 Current_Pending_Sector  0x0008   253   241   000    Old_age   Offline      -       4 
```

The Current_Pending_Sector should ideally be zero.  If not, other partitions will have issues

The Reallocated_Sector_Ct should have increased.  If its increased by more than the reduction in the Current_Pending_Sector count, the drive found more sectors it didn't like during the sync.

My view is that a drive that has a tendency to not be able to read its own writing is scrap.  If its still coverd by warranty, its a no brainer.

Check your warranty status before you add /dev/sdb3 back to the raid.

Don't forget to set up mdadm to email you about significant raid set events.

----------

## sinanqapudan

NeddySeagoon,

the procedure you describe is clear and straightforward (I've read something similar for transforming a running system into a raid1 one).

The output of

```
smartctl -a /dev/sdb
```

has not changed.

I have one further question: given that /var/log/message will not be copied into the new degraded raid array, it will end up lost. I expect that there will be no problems losing that file, but an expert's opinion (i.e. yours) on that would be appreciated.

Oh, and since we are at it, should I leave S.M.A.R.T. monitoring on on my disks once I've completed the full raid1 array restore, or is it better to shut it down? I've read cursorily (and thus I cannot further articulate on the matter) somewhere that it has a tendency to cause a strange behaviour in the disks themselves.

Thanks a lot

sinanqapudan

----------

## NeddySeagoon

sinanqapudan,

/var/log/messages will be recreated on boot if its missing.  Other than lost history, there is nothing lost.

That you appear to have made a backup of a live system may be a cause for concern. Files open for writing will be broken.

You will find that out during testing when you still have /dev/sdb, so other that learning the lesson of don't do that and your time, nothing will be lost.

As you say, the process is similar to moving a single drive install to raid1,

Leave the smart monitoring on.  The drive will do it anyway, just not report it to the BIOS at boot time.

You can run the smart monitor daemon if you wish too.

You need to either check /proc/mdstat regularly or set up mdadmd to email you when something happens to your raid set.  Doing both is a good idea too.

----------

## sinanqapudan

NeddySeagoon,

I did as you suggested but I can't boot using the new raid array as root. Boot hangs at init.early.sh.

Here's the last lines of the output when I try to boot with root on /dev/md0:

```
md: adding sdb3

md: created md126

md: bind<sdb3>

md/raid1:md126: active with 1 out of 2 mirrors

md126: detected capacity change from 0 to 161675149312

md: ... autorun DONE.

md: Loading md0: /dev/sdc3

 md0: unknown partition table

md: couldn't update array info. -22

md: could not open unknown-block(8,35).

md: md_import_device returned -16

md: could not open unknown-block(8,19).

md: md_import_device returned -16

md: starting md0 failed

EXT3-fs (md0): error: couldn't mount because of unsupported optional features (240)

EXT2-fs (md0): error: couldn't mount because of unsupported optional features (240)

EXT4-fs (md0): mounted filesystem with ordered data mode. Opts: (null)

VFS: Mounted root (ext4 filesystem) readonly on device 9:0.

Freeing unused kernel memory: 480K (c16f9000 - c1771000)

init-early.sh (902) used greatest stack depth: 6692 bytes left

init-early.sh (901) used greatest stack depth: 6256 bytes left
```

Here's the output of dmesg when I boot with root on /dev/md126

```
[   10.024363] md: Waiting for all devices to be available before autodetect

[   10.024433] md: If you don't use raid, use raid=noautodetect

[   10.024690] md: Autodetecting RAID arrays.

[   10.083502] random: nonblocking pool is initialized

[   10.099288] md: invalid raid superblock magic on sdf1

[   10.099357] md: sdf1 does not have a valid v0.90 superblock, not importing!

[   10.121174] md: invalid raid superblock magic on sdg1

[   10.121241] md: sdg1 does not have a valid v0.90 superblock, not importing!

[   10.121313] md: Scanned 6 and added 4 devices.

[   10.121379] md: autorun ...

[   10.121443] md: considering sdc3 ...

[   10.121515] md:  adding sdc3 ...

[   10.121583] md: sdc1 has different UUID to sdc3

[   10.121653] md: sdb3 has different UUID to sdc3

[   10.121721] md: sdb1 has different UUID to sdc3

[   10.121788] md: created md0

[   10.121851] md: bind<sdc3>

[   10.121928] md: running: <sdc3>

[   10.122242] md/raid1:md0: active with 1 out of 2 mirrors

[   10.122338] md0: detected capacity change from 0 to 161675149312

[   10.122457] md: considering sdc1 ...

[   10.122528] md:  adding sdc1 ...

[   10.122595] md: sdb3 has different UUID to sdc1

[   10.122667] md:  adding sdb1 ...

[   10.122916] md: created md127

[   10.122982] md: bind<sdb1>

[   10.123071] md: bind<sdc1>

[   10.123146] md: running: <sdc1><sdb1>

[   10.123452] md/raid1:md127: active with 2 out of 2 mirrors

[   10.123541] md127: detected capacity change from 0 to 104792064

[   10.123656] md: considering sdb3 ...

[   10.123727] md:  adding sdb3 ...

[   10.123957] md: created md126

[   10.124033] md: bind<sdb3>

[   10.124107] md: running: <sdb3>

[   10.124378] md/raid1:md126: active with 1 out of 2 mirrors

[   10.124465] md126: detected capacity change from 0 to 161675149312

[   10.124577] md: ... autorun DONE.

[   10.141213]  md126: unknown partition table

[   10.141433] EXT3-fs (md126): error: couldn't mount because of unsupported optional features (240)

[   10.141694] EXT2-fs (md126): error: couldn't mount because of unsupported optional features (240)

[   10.191293] EXT4-fs (md126): mounted filesystem with ordered data mode. Opts: (null)

[   10.191377] VFS: Mounted root (ext4 filesystem) readonly on device 9:126.

[   10.191797] Freeing unused kernel memory: 480K (c16f9000 - c1771000)

[   11.686062] kbd_mode (902) used greatest stack depth: 6640 bytes left

[   11.737168] loadkeys (903) used greatest stack depth: 6440 bytes left

[   11.737485] init-early.sh (901) used greatest stack depth: 6092 bytes left

[   13.989430] systemd-udevd[1066]: starting version 208

[   14.559557] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded

[   14.560035] r8169 0000:01:00.0 eth0: RTL8168d/8111d at 0xf8090000, 00:25:22:c2:a6:e4, XID 081000c0 IRQ 10

[   14.560039] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes, tx checksumming: ko]

[   14.560055] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded

[   14.560244] ACPI: PCI Interrupt Link [LNKG] enabled at IRQ 5

[   14.560261] r8169 0000:04:01.0 (unregistered net_device): not PCI Express

[   14.560548] r8169 0000:04:01.0 eth1: RTL8169sb/8110sb at 0xf8092800, 00:08:a1:c7:13:d6, XID 10000000 IRQ 5

[   14.560551] r8169 0000:04:01.0 eth1: jumbo features [frames: 7152 bytes, tx checksumming: ok]

[   15.208450]  md0: unknown partition table

[   15.248425]  md127: unknown partition table

[   15.348149] systemd-sysctl[1106]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   15.348332] systemd-sysctl[1104]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   15.348394] systemd-sysctl[1107]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   15.403990] systemd-udevd[1083]: renamed network interface eth0 to enp1s0

[   15.405148] systemd-udevd[1091]: renamed network interface eth1 to enp4s1

[   18.365587] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   18.365591] ata5.00: BMDMA stat 0x64

[   18.365597] ata5.00: failed command: READ DMA EXT

[   18.365604] ata5.00: cmd 25/00:08:20:70:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:25:70:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   18.365606] ata5.00: status: { DRDY ERR }

[   18.365608] ata5.00: error: { UNC }

[   18.433768] ata5.00: configured for UDMA/100

[   18.456777] ata5.01: configured for UDMA/100

[   18.456791] sd 4:0:0:0: [sdb] Unhandled sense code

[   18.456794] sd 4:0:0:0: [sdb]  

[   18.456796] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   18.456798] sd 4:0:0:0: [sdb]  

[   18.456799] Sense Key : Medium Error [current] [descriptor]

[   18.456805] Descriptor sense data with sense descriptors (in hex):

[   18.456806]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   18.456816]         13 15 70 25 

[   18.456821] sd 4:0:0:0: [sdb]  

[   18.456825] Add. Sense: Unrecovered read error - auto reallocate failed

[   18.456827] sd 4:0:0:0: [sdb] CDB: 

[   18.456829] Read(10): 28 00 13 15 70 20 00 00 08 00

[   18.456838] end_request: I/O error, dev sdb, sector 320172069

[   18.456841] Buffer I/O error on device sdb, logical block 40021508

[   18.456852] ata5: EH complete

[   18.551217] systemd-sysctl[1118]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   18.553325] systemd-sysctl[1117]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   23.131914] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   23.131918] ata5.00: BMDMA stat 0x64

[   23.131922] ata5.00: failed command: READ DMA EXT

[   23.131929] ata5.00: cmd 25/00:08:20:70:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:25:70:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   23.131931] ata5.00: status: { DRDY ERR }

[   23.131933] ata5.00: error: { UNC }

[   23.200786] ata5.00: configured for UDMA/100

[   23.223779] ata5.01: configured for UDMA/100

[   23.223792] sd 4:0:0:0: [sdb] Unhandled sense code

[   23.223795] sd 4:0:0:0: [sdb]  

[   23.223797] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   23.223799] sd 4:0:0:0: [sdb]  

[   23.223800] Sense Key : Medium Error [current] [descriptor]

[   23.223804] Descriptor sense data with sense descriptors (in hex):

[   23.223806]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   23.223816]         13 15 70 25 

[   23.223821] sd 4:0:0:0: [sdb]  

[   23.223825] Add. Sense: Unrecovered read error - auto reallocate failed

[   23.223827] sd 4:0:0:0: [sdb] CDB: 

[   23.223828] Read(10): 28 00 13 15 70 20 00 00 08 00

[   23.223838] end_request: I/O error, dev sdb, sector 320172069

[   23.223841] Buffer I/O error on device sdb, logical block 40021508

[   23.223850] ata5: EH complete

[   23.335692] mdadm (1108) used greatest stack depth: 5924 bytes left

[   24.964066] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   24.964070] ata5.00: BMDMA stat 0x64

[   24.964075] ata5.00: failed command: READ DMA EXT

[   24.964082] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   24.964084] ata5.00: status: { DRDY ERR }

[   24.964086] ata5.00: error: { UNC }

[   25.048763] ata5.00: configured for UDMA/100

[   25.071779] ata5.01: configured for UDMA/100

[   25.071791] sd 4:0:0:0: [sdb] Unhandled sense code

[   25.071793] sd 4:0:0:0: [sdb]  

[   25.071795] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   25.071797] sd 4:0:0:0: [sdb]  

[   25.071799] Sense Key : Medium Error [current] [descriptor]

[   25.071803] Descriptor sense data with sense descriptors (in hex):

[   25.071805]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   25.071815]         13 15 6f f0 

[   25.071820] sd 4:0:0:0: [sdb]  

[   25.071824] Add. Sense: Unrecovered read error - auto reallocate failed

[   25.071827] sd 4:0:0:0: [sdb] CDB: 

[   25.071828] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   25.071838] end_request: I/O error, dev sdb, sector 320172016

[   25.071842] Buffer I/O error on device md126, logical block 39471358

[   25.071853] ata5: EH complete

[   28.314293] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   28.314297] ata5.00: BMDMA stat 0x64

[   28.314301] ata5.00: failed command: READ DMA EXT

[   28.314308] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   28.314311] ata5.00: status: { DRDY ERR }

[   28.314313] ata5.00: error: { UNC }

[   28.398777] ata5.00: configured for UDMA/100

[   28.421779] ata5.01: configured for UDMA/100

[   28.421791] sd 4:0:0:0: [sdb] Unhandled sense code

[   28.421793] sd 4:0:0:0: [sdb]  

[   28.421796] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   28.421799] sd 4:0:0:0: [sdb]  

[   28.421800] Sense Key : Medium Error [current] [descriptor]

[   28.421804] Descriptor sense data with sense descriptors (in hex):

[   28.421806]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   28.421816]         13 15 6f f0 

[   28.421821] sd 4:0:0:0: [sdb]  

[   28.421825] Add. Sense: Unrecovered read error - auto reallocate failed

[   28.421827] sd 4:0:0:0: [sdb] CDB: 

[   28.421828] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   28.421838] end_request: I/O error, dev sdb, sector 320172016

[   28.421842] Buffer I/O error on device md126, logical block 39471358

[   28.421852] ata5: EH complete

[   30.380428] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   30.380432] ata5.00: BMDMA stat 0x64

[   30.380437] ata5.00: failed command: READ DMA EXT

[   30.380443] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   30.380446] ata5.00: status: { DRDY ERR }

[   30.380447] ata5.00: error: { UNC }

[   30.464754] ata5.00: configured for UDMA/100

[   30.487785] ata5.01: configured for UDMA/100

[   30.487798] sd 4:0:0:0: [sdb] Unhandled sense code

[   30.487801] sd 4:0:0:0: [sdb]  

[   30.487803] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   30.487805] sd 4:0:0:0: [sdb]  

[   30.487806] Sense Key : Medium Error [current] [descriptor]

[   30.487810] Descriptor sense data with sense descriptors (in hex):

[   30.487812]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   30.487822]         13 15 6f f0 

[   30.487827] sd 4:0:0:0: [sdb]  

[   30.487831] Add. Sense: Unrecovered read error - auto reallocate failed

[   30.487834] sd 4:0:0:0: [sdb] CDB: 

[   30.487835] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   30.487844] end_request: I/O error, dev sdb, sector 320172016

[   30.487848] Buffer I/O error on device md126, logical block 39471358

[   30.487860] ata5: EH complete

[   32.180551] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   32.180555] ata5.00: BMDMA stat 0x64

[   32.180559] ata5.00: failed command: READ DMA EXT

[   32.180566] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   32.180569] ata5.00: status: { DRDY ERR }

[   32.180571] ata5.00: error: { UNC }

[   32.264757] ata5.00: configured for UDMA/100

[   32.287765] ata5.01: configured for UDMA/100

[   32.287779] sd 4:0:0:0: [sdb] Unhandled sense code

[   32.287781] sd 4:0:0:0: [sdb]  

[   32.287783] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   32.287785] sd 4:0:0:0: [sdb]  

[   32.287787] Sense Key : Medium Error [current] [descriptor]

[   32.287793] Descriptor sense data with sense descriptors (in hex):

[   32.287794]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   32.287804]         13 15 6f f0 

[   32.287809] sd 4:0:0:0: [sdb]  

[   32.287813] Add. Sense: Unrecovered read error - auto reallocate failed

[   32.287816] sd 4:0:0:0: [sdb] CDB: 

[   32.287817] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   32.287827] end_request: I/O error, dev sdb, sector 320172016

[   32.287831] Buffer I/O error on device md126, logical block 39471358

[   32.287846] ata5: EH complete

[   33.738650] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   33.738654] ata5.00: BMDMA stat 0x64

[   33.738658] ata5.00: failed command: READ DMA EXT

[   33.738665] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   33.738667] ata5.00: status: { DRDY ERR }

[   33.738669] ata5.00: error: { UNC }

[   33.822759] ata5.00: configured for UDMA/100

[   33.829777] ata5.01: configured for UDMA/100

[   33.829789] sd 4:0:0:0: [sdb] Unhandled sense code

[   33.829792] sd 4:0:0:0: [sdb]  

[   33.829794] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   33.829796] sd 4:0:0:0: [sdb]  

[   33.829797] Sense Key : Medium Error [current] [descriptor]

[   33.829801] Descriptor sense data with sense descriptors (in hex):

[   33.829803]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   33.829814]         13 15 6f f0 

[   33.829819] sd 4:0:0:0: [sdb]  

[   33.829824] Add. Sense: Unrecovered read error - auto reallocate failed

[   33.829826] sd 4:0:0:0: [sdb] CDB: 

[   33.829827] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   33.829837] end_request: I/O error, dev sdb, sector 320172016

[   33.829840] Buffer I/O error on device md126, logical block 39471358

[   33.829854] ata5: EH complete

[   35.280748] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   35.280752] ata5.00: BMDMA stat 0x64

[   35.280756] ata5.00: failed command: READ DMA EXT

[   35.280763] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   35.280765] ata5.00: status: { DRDY ERR }

[   35.280767] ata5.00: error: { UNC }

[   35.364778] ata5.00: configured for UDMA/100

[   35.371780] ata5.01: configured for UDMA/100

[   35.371792] sd 4:0:0:0: [sdb] Unhandled sense code

[   35.371795] sd 4:0:0:0: [sdb]  

[   35.371797] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   35.371799] sd 4:0:0:0: [sdb]  

[   35.371800] Sense Key : Medium Error [current] [descriptor]

[   35.371804] Descriptor sense data with sense descriptors (in hex):

[   35.371806]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   35.371816]         13 15 6f f0 

[   35.371821] sd 4:0:0:0: [sdb]  

[   35.371825] Add. Sense: Unrecovered read error - auto reallocate failed

[   35.371828] sd 4:0:0:0: [sdb] CDB: 

[   35.371829] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   35.371838] end_request: I/O error, dev sdb, sector 320172016

[   35.371841] Buffer I/O error on device md126, logical block 39471358

[   35.371852] ata5: EH complete

[   39.597032] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   39.597036] ata5.00: BMDMA stat 0x64

[   39.597040] ata5.00: failed command: READ DMA EXT

[   39.597047] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   39.597050] ata5.00: status: { DRDY ERR }

[   39.597052] ata5.00: error: { UNC }

[   39.684761] ata5.00: configured for UDMA/100

[   39.707782] ata5.01: configured for UDMA/100

[   39.707795] sd 4:0:0:0: [sdb] Unhandled sense code

[   39.707798] sd 4:0:0:0: [sdb]  

[   39.707800] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   39.707802] sd 4:0:0:0: [sdb]  

[   39.707803] Sense Key : Medium Error [current] [descriptor]

[   39.707809] Descriptor sense data with sense descriptors (in hex):

[   39.707811]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   39.707821]         13 15 6f f0 

[   39.707825] sd 4:0:0:0: [sdb]  

[   39.707829] Add. Sense: Unrecovered read error - auto reallocate failed

[   39.707832] sd 4:0:0:0: [sdb] CDB: 

[   39.707833] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   39.707843] end_request: I/O error, dev sdb, sector 320172016

[   39.707847] Buffer I/O error on device md126, logical block 39471358

[   39.707861] ata5: EH complete

[   39.973584] EXT4-fs (md126): re-mounted. Opts: (null)

[   40.282629] Adding 2097148k swap on /dev/sdb2.  Priority:1 extents:1 across:2097148k FS

[   40.431304] EXT4-fs (sdd1): mounted filesystem with ordered data mode. Opts: (null)

[   43.120099] r8169 0000:01:00.0: Direct firmware load failed with error -2

[   43.120104] r8169 0000:01:00.0: Falling back to user helper

[   43.120108] r8169 0000:01:00.0 enp1s0: unable to load firmware patch rtl_nic/rtl8168d-1.fw (-2)

[   43.129346] r8169 0000:01:00.0 enp1s0: link down

[   43.129374] IPv6: ADDRCONF(NETDEV_UP): enp1s0: link is not ready

[   43.129385] r8169 0000:01:00.0 enp1s0: link down

[   45.850652] r8169 0000:01:00.0 enp1s0: link up

[   45.850665] IPv6: ADDRCONF(NETDEV_CHANGE): enp1s0: link becomes ready

[   51.203563] r8169 0000:04:01.0 enp4s1: link down

[   51.203596] IPv6: ADDRCONF(NETDEV_UP): enp4s1: link is not ready

[   51.203607] r8169 0000:04:01.0 enp4s1: link down

[   53.557250] r8169 0000:04:01.0 enp4s1: link up

[   53.557263] IPv6: ADDRCONF(NETDEV_CHANGE): enp4s1: link becomes ready

[   60.071099] type=1006 audit(1421847061.242:2): pid=2204 uid=0 old auid=4294967295 new auid=0 old ses=4294967295 new ses=1 res=1

[   62.150344] type=1006 audit(1421847063.321:3): pid=2258 uid=0 old auid=4294967295 new auid=1003 old ses=4294967295 new ses=2 res=1

[   62.160024] type=1006 audit(1421847063.330:4): pid=2262 uid=0 old auid=4294967295 new auid=1003 old ses=4294967295 new ses=3 res=1

[   71.524996] NFSD: Using /var/lib/nfs/v4recovery as the NFSv4 state recovery directory

[   71.574560] NFSD: starting 90-second grace period (net c16e07c0)

[  403.814205] type=1006 audit(1421847404.985:5): pid=2573 uid=0 old auid=4294967295 new auid=0 old ses=4294967295 new ses=4 res=1

[  545.976685] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[  545.976690] ata5.00: BMDMA stat 0x64

[  545.976694] ata5.00: failed command: READ DMA

[  545.976701] ata5.00: cmd c8/00:00:28:ef:5f/00:00:00:00:00/e3 tag 0 dma 131072 in

         res 51/40:00:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[  545.976704] ata5.00: status: { DRDY ERR }

[  545.976706] ata5.00: error: { UNC }

[  546.080781] ata5.00: configured for UDMA/100

[  546.103781] ata5.01: configured for UDMA/100

[  546.103804] sd 4:0:0:0: [sdb] Unhandled sense code

[  546.103808] sd 4:0:0:0: [sdb]  

[  546.103810] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[  546.103812] sd 4:0:0:0: [sdb]  

[  546.103814] Sense Key : Medium Error [current] [descriptor]

[  546.103818] Descriptor sense data with sense descriptors (in hex):

[  546.103819]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[  546.103829]         03 5f ef 54 

[  546.103834] sd 4:0:0:0: [sdb]  

[  546.103840] Add. Sense: Unrecovered read error - auto reallocate failed

[  546.103842] sd 4:0:0:0: [sdb] CDB: 

[  546.103843] Read(10): 28 00 03 5f ef 28 00 01 00 00

[  546.103853] end_request: I/O error, dev sdb, sector 56618836

[  546.103878] ata5: EH complete

[  549.872231] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[  549.872236] ata5.00: BMDMA stat 0x64

[  549.872241] ata5.00: failed command: READ DMA

[  549.872248] ata5.00: cmd c8/00:08:50:ef:5f/00:00:00:00:00/e3 tag 0 dma 4096 in

         res 51/40:08:54:ef:5f/40:00:13:00:00/e3 Emask 0x9 (media error)

[  549.872250] ata5.00: status: { DRDY ERR }

[  549.872252] ata5.00: error: { UNC }

[  553.699751] ata5.00: configured for UDMA/100

[  553.706782] ata5.01: configured for UDMA/100

[  553.706803] sd 4:0:0:0: [sdb] Unhandled sense code

[  553.706807] sd 4:0:0:0: [sdb]  

[  553.706809] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[  553.706811] sd 4:0:0:0: [sdb]  

[  553.706812] Sense Key : Medium Error [current] [descriptor]

[  553.706816] Descriptor sense data with sense descriptors (in hex):

[  553.706818]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[  553.706828]         03 5f ef 54 

[  553.706833] sd 4:0:0:0: [sdb]  

[  553.706838] Add. Sense: Unrecovered read error - auto reallocate failed

[  553.706841] sd 4:0:0:0: [sdb] CDB: 

[  553.706842] Read(10): 28 00 03 5f ef 50 00 00 08 00

[  553.706852] end_request: I/O error, dev sdb, sector 56618836

[  553.706870] ata5: EH complete

[ 1154.931264] type=1006 audit(1421848156.102:6): pid=2698 uid=0 old auid=4294967295 new auid=0 old ses=4294967295 new ses=5 res=1
```

I do not use initrd.

Regards

----------

## NeddySeagoon

sinanqapudan,

```
md: starting md0 failed

EXT3-fs (md0): error: couldn't mount because of unsupported optional features (240)

EXT2-fs (md0): error: couldn't mount because of unsupported optional features (240)

EXT4-fs (md0): mounted filesystem with ordered data mode. Opts: (null)

VFS: Mounted root (ext4 filesystem) readonly on device 9:0. 
```

Thats odd md0 failed to start but it got mounted anyway. The EXT3 and EXT2 errors can be ignored.

md==device 9:0.

There are no signs of mdadm scanning for raid devices either.

md: could not open unknown-block(8,35).

md: could not open unknown-block(8,19).

Thats sdc3 and sdb3

Taken together with your working boot, it suggests that your root copy is trying to use a different kernel, as it can't see the underlying block devices to create the raid set.

A photo of the failed boot may be more useful.  I assume you had to copy type that?

----------

## sinanqapudan

NeddySeagoon:

 *Quote:*   

> A photo of the failed boot may be more useful. I assume you had to copy type that?

 

You are right, I typed it from the picture.

If you need it, I can post the picture also. How can this be done?

Regards,

----------

## NeddySeagoon

sinanqapudan,

There at lots of free photo hosting sites.  Pick one, put the photo there and post the link.

Its much like a pastebin but there is no wgetpaste to help.

----------

## sinanqapudan

NeddySeagoon,

here's  the link:

http://s16.postimg.org/5h3huzn05/20150121_141334.jpg

Regards

----------

## sinanqapudan

NeddySeagoon,

before investigating the kernel issue, I would explore the possibility that I used a wrong method to copy the running system into /dev/md0. I mounted /dev/md0 on /mt and then used:

```
# rsync -avxHAS --delete --progress / /mt/
```

That was taken out of https://wiki.archlinux.org/index.php/Convert_a_single_drive_system_to_RAID#Create_the_RAID_device, but I could not use the command that was suggest as the -X option to rsync resulted in errors being reported.

Maybe you could suggest a more reliable method for restoring the system to the new raid array.

I also omitted to rsync the /boot directory as it is already part of the working raid1 array /dev/md127:

```
# cat /proc/mdstat

Personalities : [linear] [raid0] [raid1] [raid10] [raid6] [raid5] [raid4] 

md126 : active raid1 sdb3[0]

      157885888 blocks [2/1] [U_]

      

md127 : active raid1 sdc1[1] sdb1[0]

      102336 blocks [2/2] [UU]

      

md0 : active raid1 sdc3[1]

      157885888 blocks [2/1] [_U]

      

unused devices: <none>
```

The /boot/grub/grub.conf is:

```
default 0

timeout 10

splashimage=(hd1,0)/boot/grub/splash.xpm.gz

title Gentoo Linux current

root (hd1,0)

kernel /boot/current root=/dev/md126

title Gentoo Linux current md0

root (hd1,0)

kernel /boot/current root=/dev/md0
```

The device.map is:

```
(hd0)   /dev/sda

(hd1)   /dev/sdb

(hd2)   /dev/sdc

(hd3)   /dev/sdd

(hd4)   /dev/sde

(hd5)   /dev/sdf

(hd6)   /dev/sdg
```

When I boot:

```
title Gentoo Linux current md0

root (hd1,0)

kernel /boot/current root=/dev/md0
```

I get the system hanged. If I edit /boot/grub/grub.conf to:

```
default 0

timeout 10

splashimage=(hd1,0)/boot/grub/splash.xpm.gz

title Gentoo Linux current

root (hd1,0)

kernel /boot/current root=/dev/md126

title Gentoo Linux current md0

root (hd2,0)

kernel /boot/current root=/dev/md0
```

I get the Grub legacy error:

```
15 : File not found
```

which according to Grub 0.97 manual means:

 *Quote:*   

> This error is returned if the specified file name cannot be found, but everything else (like the disk/partition info) is OK. 

 

as if grub cannot find the kernel file in the /boot directory of /dev/sdc1 (hd2,0).

Hope this helps to clarify the situation

Regards

----------

## sinanqapudan

NeddySeagoon,

I managed to boot with /dev/md0 as boot.

I added the rw option to the grub boot line as per https://forums.gentoo.org/viewtopic-p-6701287.html?sid=3f8e66a550297ea5876c334813364c2c

The system was slower than usual to boot (i.e. slower than when I boot using /dev/md126), due to the time it took to assemble /dev/md0, but in the end it booted.

Still the system does not seem to be OK. For a start, SSH is not working properly.

When I execute:

```
root@host
```

I get:

```
Password: 

PTY allocation request failed on channel 0
```

When I execute:

```
ssh root@host "/bin/bash -i"
```

I get:

```
$ ssh root@host "/bin/bash -i"

Password: 

bash: cannot set terminal process group (-1): Inappropriate ioctl for device

bash: no job control in this shell

root@sargon ~ #

```

It seems to be a step forward, but not sure whether it actually is.

Regards

----------

## sinanqapudan

post deletedLast edited by sinanqapudan on Thu Jan 22, 2015 4:01 pm; edited 1 time in total

----------

## sinanqapudan

NeddySeagoon,

further updating on the issue, I stopped the system, tried to reboot without the rw option and the system hanged as before.

I then rebooted with the rw option and the system was able to boot. SSH is now working. All this appears to be highly suspect.

Here's dmesg (abridged):

```
[    0.284067] mousedev: PS/2 mouse device common for all mice

[    0.284418] md: linear personality registered for level -1

[    0.284487] md: raid0 personality registered for level 0

[    0.284554] md: raid1 personality registered for level 1

[    0.284621] md: raid10 personality registered for level 10

[    0.284742] md: raid6 personality registered for level 6

[    0.284809] md: raid5 personality registered for level 5

[    0.284876] md: raid4 personality registered for level 4

[    0.285043] hidraw: raw HID events driver (C) Jiri Kosina

[    0.285611] usbcore: registered new interface driver usbhid

[    0.285679] usbhid: USB HID core driver

[    0.285888] TCP: bic registered

[    0.285953] TCP: cubic registered

[    0.286030] Initializing XFRM netlink socket

[    0.286524] NET: Registered protocol family 10

[    0.287076] sit: IPv6 over IPv4 tunneling driver

[    0.287366] NET: Registered protocol family 17

[    0.287439] NET: Registered protocol family 15

[    0.287528] Key type dns_resolver registered

[    0.288510]   Magic number: 15:242:589

[    0.288646] console [netcon0] enabled

[    0.288711] netconsole: network logging started

[    0.376054] hub 1-0:1.0: state 7 ports 2 chg 0000 evt 0000

[    0.376065] hub 1-0:1.0: hub_suspend

[    0.376076] usb usb1: bus auto-suspend, wakeup 1

[    0.376080] usb usb1: suspend_rh

[    0.377033] uhci_hcd 0000:00:1d.1: port 2 portsc 01ab,00

[    0.377040] hub 2-0:1.0: port 2: status 0301 change 0003

[    0.378036] hub 3-0:1.0: state 7 ports 2 chg 0000 evt 0000

[    0.378042] hub 3-0:1.0: hub_suspend

[    0.378050] usb usb3: bus auto-suspend, wakeup 1

[    0.378052] usb usb3: suspend_rh

[    0.380040] hub 4-0:1.0: state 7 ports 2 chg 0000 evt 0000

[    0.380045] hub 4-0:1.0: hub_suspend

[    0.380052] usb usb4: bus auto-suspend, wakeup 1

[    0.380055] usb usb4: suspend_rh

[    0.429271] ata8.00: ATA-9: WDC WD30EZRX-00SPEB0, 80.00A80, max UDMA/133

[    0.429343] ata8.00: 5860533168 sectors, multi 16: LBA48 NCQ (depth 0/32)

[    0.429553] ata8.01: ATA-9: WDC WD30EZRX-00SPEB0, 80.00A80, max UDMA/133

[    0.429624] ata8.01: 5860533168 sectors, multi 16: LBA48 NCQ (depth 0/32)

[    0.429900] ata5.00: ATA-7: Maxtor 6B160P0, BAH41B70, max UDMA/133

[    0.429971] ata5.00: 320173056 sectors, multi 16: LBA48 

[    0.430421] ata8.00: configured for UDMA/133

[    0.431635] ata5.01: ATA-7: Maxtor 6B160P0, BAH41B70, max UDMA/133

[    0.431704] ata5.01: 320173056 sectors, multi 16: LBA48 

[    0.433265] ata8.01: configured for UDMA/133

[    0.438757] ata5.00: configured for UDMA/100

[    0.445755] ata5.01: configured for UDMA/100

[    0.477028] hub 2-0:1.0: state 7 ports 2 chg 0004 evt 0000

[    0.477042] hub 2-0:1.0: port 2, status 0301, change 0000, 1.5 Mb/s

[    0.579028] usb 2-2: new low-speed USB device number 2 using uhci_hcd

[    0.724666] usb 2-2: skipped 1 descriptor after interface

[    0.724669] usb 2-2: skipped 1 descriptor after interface

[    0.729662] usb 2-2: default language 0x0409

[    0.744667] usb 2-2: udev 2, busnum 2, minor = 129

[    0.744671] usb 2-2: New USB device found, idVendor=046d, idProduct=c517

[    0.744741] usb 2-2: New USB device strings: Mfr=1, Product=2, SerialNumber=0

[    0.744812] usb 2-2: Product: USB Receiver

[    0.744878] usb 2-2: Manufacturer: Logitech

[    0.745090] usb 2-2: usb_probe_device

[    0.745094] usb 2-2: configuration #1 chosen from 1 choice

[    0.747675] usb 2-2: adding 2-2:1.0 (config #1, interface 0)

[    0.747746] usbhid 2-2:1.0: usb_probe_interface

[    0.747749] usbhid 2-2:1.0: usb_probe_interface - got id

[    0.762458] input: Logitech USB Receiver as /devices/pci0000:00/0000:00:1d.1/usb2/2-2/2-2:1.0/input/input5

[    0.762585] uhci_hcd 0000:00:1d.1: reserve dev 2 ep81-INT, period 8, phase 4, 118 us

[    0.762732] logitech 0003:046D:C517.0001: input,hidraw0: USB HID v1.10 Keyboard [Logitech USB Receiver] on usb-0000:00:1d.1-2/input0

[    0.762869] usb 2-2: adding 2-2:1.1 (config #1, interface 1)

[    0.762934] usbhid 2-2:1.1: usb_probe_interface

[    0.762937] usbhid 2-2:1.1: usb_probe_interface - got id

[    0.796745] logitech 0003:046D:C517.0002: fixing up Logitech keyboard report descriptor

[    0.798890] input: Logitech USB Receiver as /devices/pci0000:00/0000:00:1d.1/usb2/2-2/2-2:1.1/input/input6

[    0.799022] uhci_hcd 0000:00:1d.1: reserve dev 2 ep82-INT, period 8, phase 4, 118 us

[    0.799431] usbhid 2-2:1.1: looking for a minor, starting at 96

[    0.799565] logitech 0003:046D:C517.0002: input,hiddev0,hidraw1: USB HID v1.10 Mouse [Logitech USB Receiver] on usb-0000:00:1d.1-2/input1

[    1.232024] tsc: Refined TSC clocksource calibration: 2792.999 MHz

[    1.504242] ata7.00: ATA-8: WDC WD20EARS-00MVWB0, 51.0AB51, max UDMA/133

[    1.504314] ata7.00: 3907029168 sectors, multi 16: LBA48 NCQ (depth 0/32)

[    1.592190] ata7.01: ATA-8: WDC WD20EARS-00MVWB0, 51.0AB51, max UDMA/133

[    1.592260] ata7.01: 3907029168 sectors, multi 16: LBA48 NCQ (depth 0/32)

[    1.669256] ata7.00: configured for UDMA/133

[    1.689269] ata7.01: configured for UDMA/133

[    2.232041] Switched to clocksource tsc

[    2.288031] ata1: SATA link down (SStatus 0 SControl 0)

[    4.308029] ata2: SATA link down (SStatus 0 SControl 0)

[    6.328028] ata3: SATA link down (SStatus 0 SControl 0)

[    8.460034] ata4: SATA link up 3.0 Gbps (SStatus 123 SControl 0)

[    8.466438] ata4.00: ATA-7: SAMSUNG HD103UJ, 1AA01113, max UDMA7

[    8.466508] ata4.00: 1953525168 sectors, multi 16: LBA48 NCQ (depth 31/32)

[    8.472978] ata4.00: configured for UDMA/100

[    8.473175] scsi 3:0:0:0: Direct-Access     ATA      SAMSUNG HD103UJ  1AA0 PQ: 0 ANSI: 5

[    8.473522] sd 3:0:0:0: [sda] 1953525168 512-byte logical blocks: (1.00 TB/931 GiB)

[    8.473657] sd 3:0:0:0: [sda] Write Protect is off

[    8.473726] sd 3:0:0:0: [sda] Mode Sense: 00 3a 00 00

[    8.473756] sd 3:0:0:0: [sda] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    8.474339] scsi 4:0:0:0: Direct-Access     ATA      Maxtor 6B160P0   BAH4 PQ: 0 ANSI: 5

[    8.474650] sd 4:0:0:0: [sdb] 320173056 512-byte logical blocks: (163 GB/152 GiB)

[    8.474780] sd 4:0:0:0: [sdb] Write Protect is off

[    8.474848] sd 4:0:0:0: [sdb] Mode Sense: 00 3a 00 00

[    8.474875] sd 4:0:0:0: [sdb] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    8.483250]  sda: sda1

[    8.483629] sd 3:0:0:0: [sda] Attached SCSI disk

[    8.494639]  sdb: sdb1 sdb2 sdb3

[    8.494769] scsi 4:0:1:0: Direct-Access     ATA      Maxtor 6B160P0   BAH4 PQ: 0 ANSI: 5

[    8.495279] sd 4:0:1:0: [sdc] 320173056 512-byte logical blocks: (163 GB/152 GiB)

[    8.495410] sd 4:0:1:0: [sdc] Write Protect is off

[    8.495479] sd 4:0:1:0: [sdc] Mode Sense: 00 3a 00 00

[    8.495506] sd 4:0:1:0: [sdc] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    8.496212] scsi 6:0:0:0: Direct-Access     ATA      WDC WD20EARS-00M 51.0 PQ: 0 ANSI: 5

[    8.496522] sd 6:0:0:0: [sdd] 3907029168 512-byte logical blocks: (2.00 TB/1.81 TiB)

[    8.496594] sd 6:0:0:0: [sdd] 4096-byte physical blocks

[    8.496721] sd 6:0:0:0: [sdd] Write Protect is off

[    8.496789] sd 6:0:0:0: [sdd] Mode Sense: 00 3a 00 00

[    8.496816] sd 6:0:0:0: [sdd] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    8.512974]  sdc: sdc1 sdc2 sdc3

[    8.513184] sd 4:0:0:0: [sdb] Attached SCSI disk

[    8.513658] sd 4:0:1:0: [sdc] Attached SCSI disk

[    8.516375]  sdd: sdd1

[    8.516501] scsi 6:0:1:0: Direct-Access     ATA      WDC WD20EARS-00M 51.0 PQ: 0 ANSI: 5

[    8.516996] sd 6:0:0:0: [sdd] Attached SCSI disk

[    8.517167] sd 6:0:1:0: [sde] 3907029168 512-byte logical blocks: (2.00 TB/1.81 TiB)

[    8.517298] sd 6:0:1:0: [sde] Write Protect is off

[    8.517366] sd 6:0:1:0: [sde] Mode Sense: 00 3a 00 00

[    8.517393] sd 6:0:1:0: [sde] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    8.517943] scsi 7:0:0:0: Direct-Access     ATA      WDC WD30EZRX-00S 80.0 PQ: 0 ANSI: 5

[    8.518266] sd 7:0:0:0: [sdf] 5860533168 512-byte logical blocks: (3.00 TB/2.72 TiB)

[    8.518339] sd 7:0:0:0: [sdf] 4096-byte physical blocks

[    8.518466] sd 7:0:0:0: [sdf] Write Protect is off

[    8.518536] sd 7:0:0:0: [sdf] Mode Sense: 00 3a 00 00

[    8.518562] sd 7:0:0:0: [sdf] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    8.533500]  sde: unknown partition table

[    8.533781] sd 6:0:1:0: [sde] Attached SCSI disk

[    9.267666]  sdf: sdf1

[    9.267793] scsi 7:0:1:0: Direct-Access     ATA      WDC WD30EZRX-00S 80.0 PQ: 0 ANSI: 5

[    9.268291] sd 7:0:0:0: [sdf] Attached SCSI disk

[    9.268445] sd 7:0:1:0: [sdg] 5860533168 512-byte logical blocks: (3.00 TB/2.72 TiB)

[    9.268517] sd 7:0:1:0: [sdg] 4096-byte physical blocks

[    9.268643] sd 7:0:1:0: [sdg] Write Protect is off

[    9.268711] sd 7:0:1:0: [sdg] Mode Sense: 00 3a 00 00

[    9.268738] sd 7:0:1:0: [sdg] Write cache: enabled, read cache: enabled, doesn't support DPO or FUA

[    9.976701]  sdg: sdg1

[    9.977066] sd 7:0:1:0: [sdg] Attached SCSI disk

[    9.977145] md: Waiting for all devices to be available before autodetect

[    9.977214] md: If you don't use raid, use raid=noautodetect

[    9.977472] md: Autodetecting RAID arrays.

[   10.032614] random: nonblocking pool is initialized

[   10.052746] md: invalid raid superblock magic on sdf1

[   10.052814] md: sdf1 does not have a valid v0.90 superblock, not importing!

[   10.073958] md: invalid raid superblock magic on sdg1

[   10.074038] md: sdg1 does not have a valid v0.90 superblock, not importing!

[   10.074111] md: Scanned 6 and added 4 devices.

[   10.074177] md: autorun ...

[   10.074242] md: considering sdc3 ...

[   10.074315] md:  adding sdc3 ...

[   10.074382] md: sdc1 has different UUID to sdc3

[   10.074450] md: sdb3 has different UUID to sdc3

[   10.074518] md: sdb1 has different UUID to sdc3

[   10.074584] md: created md0

[   10.074648] md: bind<sdc3>

[   10.074723] md: running: <sdc3>

[   10.075037] md/raid1:md0: active with 1 out of 2 mirrors

[   10.075131] md0: detected capacity change from 0 to 161675149312

[   10.075249] md: considering sdc1 ...

[   10.075320] md:  adding sdc1 ...

[   10.075388] md: sdb3 has different UUID to sdc1

[   10.075459] md:  adding sdb1 ...

[   10.075708] md: created md127

[   10.075773] md: bind<sdb1>

[   10.075848] md: bind<sdc1>

[   10.075922] md: running: <sdc1><sdb1>

[   10.076239] md/raid1:md127: active with 2 out of 2 mirrors

[   10.076328] md127: detected capacity change from 0 to 104792064

[   10.076442] md: considering sdb3 ...

[   10.076512] md:  adding sdb3 ...

[   10.076743] md: created md126

[   10.076809] md: bind<sdb3>

[   10.076883] md: running: <sdb3>

[   10.077165] md/raid1:md126: active with 1 out of 2 mirrors

[   10.077253] md126: detected capacity change from 0 to 161675149312

[   10.077365] md: ... autorun DONE.

[   10.097400]  md0: unknown partition table

[   10.097619] EXT3-fs (md0): error: couldn't mount because of unsupported optional features (240)

[   10.097896] EXT2-fs (md0): error: couldn't mount because of unsupported optional features (240)

[   10.172230] EXT4-fs (md0): mounted filesystem with ordered data mode. Opts: (null)

[   10.172311] VFS: Mounted root (ext4 filesystem) on device 9:0.

[   10.172727] Freeing unused kernel memory: 480K (c16f9000 - c1771000)

[   11.339818] kbd_mode (902) used greatest stack depth: 6440 bytes left

[   11.407787] loadkeys (903) used greatest stack depth: 6420 bytes left

[   11.408166] init-early.sh (901) used greatest stack depth: 6256 bytes left

[   12.786777] systemd-udevd[1050]: starting version 208

[   13.130736] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130749] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130759] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130768] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130777] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130786] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130795] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130804] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130813] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130822] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130831] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130839] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.130848] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument

[   13.168332] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded

[   13.168913] r8169 0000:01:00.0 eth0: RTL8168d/8111d at 0xf807a000, 00:25:22:c2:a6:e4, XID 081000c0 IRQ 10

[   13.168916] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes, tx checksumming: ko]

[   13.168932] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded

[   13.169136] ACPI: PCI Interrupt Link [LNKG] enabled at IRQ 5

[   13.169154] r8169 0000:04:01.0 (unregistered net_device): not PCI Express

[   13.169443] r8169 0000:04:01.0 eth1: RTL8169sb/8110sb at 0xf807c800, 00:08:a1:c7:13:d6, XID 10000000 IRQ 5

[   13.169447] r8169 0000:04:01.0 eth1: jumbo features [frames: 7152 bytes, tx checksumming: ok]

[   16.294768] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   16.294772] ata5.00: BMDMA stat 0x64

[   16.294777] ata5.00: failed command: READ DMA EXT

[   16.294784] ata5.00: cmd 25/00:08:20:70:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:25:70:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   16.294787] ata5.00: status: { DRDY ERR }

[   16.294788] ata5.00: error: { UNC }

[   16.378764] ata5.00: configured for UDMA/100

[   16.401768] ata5.01: configured for UDMA/100

[   16.401784] sd 4:0:0:0: [sdb] Unhandled sense code

[   16.401786] sd 4:0:0:0: [sdb]  

[   16.401788] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   16.401790] sd 4:0:0:0: [sdb]  

[   16.401792] Sense Key : Medium Error [current] [descriptor]

[   16.401796] Descriptor sense data with sense descriptors (in hex):

[   16.401797]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   16.401807]         13 15 70 25 

[   16.401812] sd 4:0:0:0: [sdb]  

[   16.401818] Add. Sense: Unrecovered read error - auto reallocate failed

[   16.401820] sd 4:0:0:0: [sdb] CDB: 

[   16.401822] Read(10): 28 00 13 15 70 20 00 00 08 00

[   16.401831] end_request: I/O error, dev sdb, sector 320172069

[   16.401834] Buffer I/O error on device sdb, logical block 40021508

[   16.401845] ata5: EH complete

[   16.501614] systemd-udevd[1072]: renamed network interface eth0 to enp1s0

[   16.513646] systemd-udevd[1069]: renamed network interface eth1 to enp4s1

[   19.369999] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   19.370015] ata5.00: BMDMA stat 0x64

[   19.370019] ata5.00: failed command: READ DMA EXT

[   19.370026] ata5.00: cmd 25/00:08:20:70:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:25:70:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   19.370029] ata5.00: status: { DRDY ERR }

[   19.370030] ata5.00: error: { UNC }

[   19.454770] ata5.00: configured for UDMA/100

[   19.477782] ata5.01: configured for UDMA/100

[   19.477798] sd 4:0:0:0: [sdb] Unhandled sense code

[   19.477802] sd 4:0:0:0: [sdb]  

[   19.477804] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   19.477806] sd 4:0:0:0: [sdb]  

[   19.477807] Sense Key : Medium Error [current] [descriptor]

[   19.477812] Descriptor sense data with sense descriptors (in hex):

[   19.477813]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   19.477823]         13 15 70 25 

[   19.477828] sd 4:0:0:0: [sdb]  

[   19.477832] Add. Sense: Unrecovered read error - auto reallocate failed

[   19.477835] sd 4:0:0:0: [sdb] CDB: 

[   19.477836] Read(10): 28 00 13 15 70 20 00 00 08 00

[   19.477845] end_request: I/O error, dev sdb, sector 320172069

[   19.477849] Buffer I/O error on device sdb, logical block 40021508

[   19.477867] ata5: EH complete

[   19.751637]  md127: unknown partition table

[   19.809446]  md126: unknown partition table

[   19.852471] systemd-sysctl[1090]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   19.852613] systemd-sysctl[1091]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   19.852665] systemd-sysctl[1089]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   19.852804] systemd-sysctl[1100]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   19.852928] systemd-sysctl[1101]: Overwriting earlier assignment of kernel/sysrq in file '/usr/lib/sysctl.d/60-gentoo.conf'.

[   20.003236] mdadm (1094) used greatest stack depth: 6220 bytes left

[   21.910206] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   21.910210] ata5.00: BMDMA stat 0x64

[   21.910215] ata5.00: failed command: READ DMA EXT

[   21.910223] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   21.910226] ata5.00: status: { DRDY ERR }

[   21.910227] ata5.00: error: { UNC }

[   21.994783] ata5.00: configured for UDMA/100

[   22.017758] ata5.01: configured for UDMA/100

[   22.017771] sd 4:0:0:0: [sdb] Unhandled sense code

[   22.017773] sd 4:0:0:0: [sdb]  

[   22.017775] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   22.017777] sd 4:0:0:0: [sdb]  

[   22.017779] Sense Key : Medium Error [current] [descriptor]

[   22.017783] Descriptor sense data with sense descriptors (in hex):

[   22.017784]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   22.017795]         13 15 6f f0 

[   22.017799] sd 4:0:0:0: [sdb]  

[   22.017803] Add. Sense: Unrecovered read error - auto reallocate failed

[   22.017806] sd 4:0:0:0: [sdb] CDB: 

[   22.017807] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   22.017816] end_request: I/O error, dev sdb, sector 320172016

[   22.017820] Buffer I/O error on device md126, logical block 39471358

[   22.017833] ata5: EH complete

[   23.677332] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   23.677337] ata5.00: BMDMA stat 0x64

[   23.677341] ata5.00: failed command: READ DMA EXT

[   23.677348] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   23.677350] ata5.00: status: { DRDY ERR }

[   23.677352] ata5.00: error: { UNC }

[   23.761807] ata5.00: configured for UDMA/100

[   23.768805] ata5.01: configured for UDMA/100

[   23.768817] sd 4:0:0:0: [sdb] Unhandled sense code

[   23.768820] sd 4:0:0:0: [sdb]  

[   23.768822] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   23.768824] sd 4:0:0:0: [sdb]  

[   23.768826] Sense Key : Medium Error [current] [descriptor]

[   23.768830] Descriptor sense data with sense descriptors (in hex):

[   23.768832]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   23.768842]         13 15 6f f0 

[   23.768847] sd 4:0:0:0: [sdb]  

[   23.768854] Add. Sense: Unrecovered read error - auto reallocate failed

[   23.768856] sd 4:0:0:0: [sdb] CDB: 

[   23.768857] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   23.768867] end_request: I/O error, dev sdb, sector 320172016

[   23.768871] Buffer I/O error on device md126, logical block 39471358

[   23.768883] ata5: EH complete

[   25.518467] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   25.518471] ata5.00: BMDMA stat 0x64

[   25.518476] ata5.00: failed command: READ DMA EXT

[   25.518482] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   25.518485] ata5.00: status: { DRDY ERR }

[   25.518487] ata5.00: error: { UNC }

[   25.605748] ata5.00: configured for UDMA/100

[   25.628743] ata5.01: configured for UDMA/100

[   25.628757] sd 4:0:0:0: [sdb] Unhandled sense code

[   25.628759] sd 4:0:0:0: [sdb]  

[   25.628761] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   25.628763] sd 4:0:0:0: [sdb]  

[   25.628765] Sense Key : Medium Error [current] [descriptor]

[   25.628769] Descriptor sense data with sense descriptors (in hex):

[   25.628770]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   25.628781]         13 15 6f f0 

[   25.628785] sd 4:0:0:0: [sdb]  

[   25.628791] Add. Sense: Unrecovered read error - auto reallocate failed

[   25.628794] sd 4:0:0:0: [sdb] CDB: 

[   25.628795] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   25.628804] end_request: I/O error, dev sdb, sector 320172016

[   25.628807] Buffer I/O error on device md126, logical block 39471358

[   25.628818] ata5: EH complete

[   28.701681] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   28.701684] ata5.00: BMDMA stat 0x64

[   28.701687] ata5.00: failed command: READ DMA EXT

[   28.701693] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   28.701696] ata5.00: status: { DRDY ERR }

[   28.701698] ata5.00: error: { UNC }

[   28.785783] ata5.00: configured for UDMA/100

[   28.792766] ata5.01: configured for UDMA/100

[   28.792777] sd 4:0:0:0: [sdb] Unhandled sense code

[   28.792779] sd 4:0:0:0: [sdb]  

[   28.792781] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   28.792783] sd 4:0:0:0: [sdb]  

[   28.792784] Sense Key : Medium Error [current] [descriptor]

[   28.792787] Descriptor sense data with sense descriptors (in hex):

[   28.792788]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   28.792798]         13 15 6f f0 

[   28.792803] sd 4:0:0:0: [sdb]  

[   28.792806] Add. Sense: Unrecovered read error - auto reallocate failed

[   28.792808] sd 4:0:0:0: [sdb] CDB: 

[   28.792810] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   28.792818] end_request: I/O error, dev sdb, sector 320172016

[   28.792821] Buffer I/O error on device md126, logical block 39471358

[   28.792831] ata5: EH complete

[   32.009921] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   32.009927] ata5.00: BMDMA stat 0x64

[   32.009931] ata5.00: failed command: READ DMA EXT

[   32.009938] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   32.009940] ata5.00: status: { DRDY ERR }

[   32.009942] ata5.00: error: { UNC }

[   32.094770] ata5.00: configured for UDMA/100

[   32.101770] ata5.01: configured for UDMA/100

[   32.101783] sd 4:0:0:0: [sdb] Unhandled sense code

[   32.101785] sd 4:0:0:0: [sdb]  

[   32.101787] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   32.101789] sd 4:0:0:0: [sdb]  

[   32.101790] Sense Key : Medium Error [current] [descriptor]

[   32.101794] Descriptor sense data with sense descriptors (in hex):

[   32.101796]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   32.101808]         13 15 6f f0 

[   32.101812] sd 4:0:0:0: [sdb]  

[   32.101817] Add. Sense: Unrecovered read error - auto reallocate failed

[   32.101819] sd 4:0:0:0: [sdb] CDB: 

[   32.101820] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   32.101831] end_request: I/O error, dev sdb, sector 320172016

[   32.101835] Buffer I/O error on device md126, logical block 39471358

[   32.101844] ata5: EH complete

[   35.193130] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   35.193133] ata5.00: BMDMA stat 0x64

[   35.193135] ata5.00: failed command: READ DMA EXT

[   35.193142] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   35.193145] ata5.00: status: { DRDY ERR }

[   35.193146] ata5.00: error: { UNC }

[   35.277782] ata5.00: configured for UDMA/100

[   35.284766] ata5.01: configured for UDMA/100

[   35.284774] sd 4:0:0:0: [sdb] Unhandled sense code

[   35.284777] sd 4:0:0:0: [sdb]  

[   35.284778] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   35.284780] sd 4:0:0:0: [sdb]  

[   35.284781] Sense Key : Medium Error [current] [descriptor]

[   35.284785] Descriptor sense data with sense descriptors (in hex):

[   35.284786]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   35.284796]         13 15 6f f0 

[   35.284800] sd 4:0:0:0: [sdb]  

[   35.284803] Add. Sense: Unrecovered read error - auto reallocate failed

[   35.284806] sd 4:0:0:0: [sdb] CDB: 

[   35.284807] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   35.284816] end_request: I/O error, dev sdb, sector 320172016

[   35.284820] Buffer I/O error on device md126, logical block 39471358

[   35.284831] ata5: EH complete

[   36.801250] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   36.801254] ata5.00: BMDMA stat 0x64

[   36.801258] ata5.00: failed command: READ DMA EXT

[   36.801265] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   36.801268] ata5.00: status: { DRDY ERR }

[   36.801270] ata5.00: error: { UNC }

[   36.885742] ata5.00: configured for UDMA/100

[   36.892780] ata5.01: configured for UDMA/100

[   36.892792] sd 4:0:0:0: [sdb] Unhandled sense code

[   36.892794] sd 4:0:0:0: [sdb]  

[   36.892796] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   36.892799] sd 4:0:0:0: [sdb]  

[   36.892800] Sense Key : Medium Error [current] [descriptor]

[   36.892804] Descriptor sense data with sense descriptors (in hex):

[   36.892807]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   36.892817]         13 15 6f f0 

[   36.892821] sd 4:0:0:0: [sdb]  

[   36.892826] Add. Sense: Unrecovered read error - auto reallocate failed

[   36.892828] sd 4:0:0:0: [sdb] CDB: 

[   36.892829] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   36.892840] end_request: I/O error, dev sdb, sector 320172016

[   36.892843] Buffer I/O error on device md126, logical block 39471358

[   36.892852] ata5: EH complete

[   39.968455] ata5.00: exception Emask 0x0 SAct 0x0 SErr 0x0 action 0x0

[   39.968457] ata5.00: BMDMA stat 0x64

[   39.968460] ata5.00: failed command: READ DMA EXT

[   39.968467] ata5.00: cmd 25/00:08:f0:6f:15/00:00:13:00:00/e0 tag 0 dma 4096 in

         res 51/40:08:f0:6f:15/40:00:13:00:00/e0 Emask 0x9 (media error)

[   39.968469] ata5.00: status: { DRDY ERR }

[   39.968471] ata5.00: error: { UNC }

[   40.052757] ata5.00: configured for UDMA/100

[   40.059771] ata5.01: configured for UDMA/100

[   40.059780] sd 4:0:0:0: [sdb] Unhandled sense code

[   40.059782] sd 4:0:0:0: [sdb]  

[   40.059784] Result: hostbyte=DID_OK driverbyte=DRIVER_SENSE

[   40.059786] sd 4:0:0:0: [sdb]  

[   40.059787] Sense Key : Medium Error [current] [descriptor]

[   40.059790] Descriptor sense data with sense descriptors (in hex):

[   40.059792]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00 

[   40.059801]         13 15 6f f0 

[   40.059806] sd 4:0:0:0: [sdb]  

[   40.059809] Add. Sense: Unrecovered read error - auto reallocate failed

[   40.059811] sd 4:0:0:0: [sdb] CDB: 

[   40.059812] Read(10): 28 00 13 15 6f f0 00 00 08 00

[   40.059821] end_request: I/O error, dev sdb, sector 320172016

[   40.059824] Buffer I/O error on device md126, logical block 39471358

[   40.059835] ata5: EH complete

[   41.983465] Adding 2097148k swap on /dev/sdc2.  Priority:1 extents:1 across:2097148k FS

[   41.985673] Adding 2097148k swap on /dev/sdb2.  Priority:1 extents:1 across:2097148k FS

[   42.138047] EXT4-fs (sdd1): mounted filesystem with ordered data mode. Opts: (null)
```

When it reached:

```
[   10.172727] Freeing unused kernel memory: 480K (c16f9000 - c1771000)

[   11.339818] kbd_mode (902) used greatest stack depth: 6440 bytes left

[   11.407787] loadkeys (903) used greatest stack depth: 6420 bytes left

[   11.408166] init-early.sh (901) used greatest stack depth: 6256 bytes left
```

it stopped for a while but I could hear the disk working and after some 15 seconds the boot process resumed.

Do you have any suggestions on how to repair the system?

Thanks

----------

## NeddySeagoon

sinanqapudan,

Is the pause repeatable or was the system doing something like a rootfsck?

What do you have in dmesg once it boots?

----------

## sinanqapudan

NeddySeagoon,

 *Quote:*   

> Is the pause repeatable or was the system doing something like a rootfsck? 

 

I'm not sure of what you mean. Could you please expand a bit on the matter?

Dmesg output is in my previous post.

Regards[/code]

----------

## NeddySeagoon

sinanqapudan,

Does the pause happen every boot or was it just the first time, when some process lixe a fsck happened that does not happen every time?

If you have timestamps on dmesg, can you see what caused the delay? 

If timestamps are off, read about printk.time= in /usr/src/linux/Documentation/kernel-parameters.txt

You can also set it on by rebuilding your kernel.

-- edit ---

I must be having a senior moment ...

```
[   19.477807] Sense Key : Medium Error [current] [descriptor]

[   19.477812] Descriptor sense data with sense descriptors (in hex):

[   19.477813]         72 03 11 04 00 00 00 0c 00 0a 80 00 00 00 00 00

[   19.477823]         13 15 70 25

[   19.477828] sd 4:0:0:0: [sdb] 

[   19.477832] Add. Sense: Unrecovered read error - auto reallocate failed

[   19.477835] sd 4:0:0:0: [sdb] CDB:

[   19.477836] Read(10): 28 00 13 15 70 20 00 00 08 00

[   19.477845] end_request: I/O error, dev sdb, sector 320172069

[   19.477849] Buffer I/O error on device sdb, logical block 40021508

[   19.477867] ata5: EH complete
```

 will be delaying things.

You still have the old raid connected.  The controller still has several goes at reading the faulty sector on sdb, which is a bit odd as you should be boot and using root from the recovered degraded set.

Ahhh...  but its a different block, its now sector 320172069

----------

## sinanqapudan

NeddySeagoon.

I booted two times and the pause happened both times. One thing worth noting is that i forgot to:

```
# mdadm --detail --scan >> /etc/mdadm.conf
```

in the /etc/mdadm.conf of the new array.

I have timestamps in dmesg but they do not represent correctly what I experienced, or at least what was the output visible on the screen.

The system (or rather the screen output) hung at:

```
[   11.408166] init-early.sh (901) used greatest stack depth: 6256 bytes left
```

what follows in dmesg is:

```
[   12.786777] systemd-udevd[1050]: starting version 208 

[   13.130736] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130749] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130759] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130768] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130777] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130786] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130795] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130804] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130813] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130822] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130831] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130839] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.130848] systemd-udevd[1069]: Error calling EVIOCSKEYCODE: Invalid argument 

[   13.168332] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded 

[   13.168913] r8169 0000:01:00.0 eth0: RTL8168d/8111d at 0xf807a000, 00:25:22:c2:a6:e4, XID 081000c0 IRQ 10 

[   13.168916] r8169 0000:01:00.0 eth0: jumbo features [frames: 9200 bytes, tx checksumming: ko] 

[   13.168932] r8169 Gigabit Ethernet driver 2.3LK-NAPI loaded 

[   13.169136] ACPI: PCI Interrupt Link [LNKG] enabled at IRQ 5 
```

and so on.

Now the system **appears** to be working fine.

Regards

----------

## sinanqapudan

Ok, I'm back from my holidays.

I changed my kernel .config to

```
CONFIG_DEVTMPFS=y 

CONFIG_DEVTMPFS_MOUNT=y 
```

previously it was

```
CONFIG_DEVTMPFS=y 

CONFIG_DEVTMPFS_MOUNT=n 
```

and now the system boot using md0 as root in readonly mode.

I have to say that I'm a bit perplexed by this. Why on earth should the same system be able to boot readonly from one disk and require read&write from another it's a mistery to me. One could argue that the two sytems are not *exactly* the same, but I think that it is not to the point.

Anyway thanks a lot for your support, NeddySeagoon.

----------

## sinanqapudan

Hi there,

I've added dev/sdb3 to the raid array and the syncing is now completed:

The output of 

```
smartctl -s /dev/sdb3
```

is:

```
ID# ATTRIBUTE_NAME          FLAG     VALUE WORST THRESH TYPE      UPDATED  WHEN_FAILED RAW_VALUE

  3 Spin_Up_Time            0x0027   213   203   063    Pre-fail  Always       -       12041

  4 Start_Stop_Count        0x0032   253   253   000    Old_age   Always       -       1823

  5 Reallocated_Sector_Ct   0x0033   252   240   063    Pre-fail  Always       -       15

  6 Read_Channel_Margin     0x0001   253   253   100    Pre-fail  Offline      -       0

  7 Seek_Error_Rate         0x000a   253   252   000    Old_age   Always       -       0

  8 Seek_Time_Performance   0x0027   239   226   187    Pre-fail  Always       -       53855

  9 Power_On_Minutes        0x0032   176   176   000    Old_age   Always       -       783h+10m

 10 Spin_Retry_Count        0x002b   253   252   157    Pre-fail  Always       -       0

 11 Calibration_Retry_Count 0x002b   253   252   223    Pre-fail  Always       -       0

 12 Power_Cycle_Count       0x0032   247   247   000    Old_age   Always       -       2577

192 Power-Off_Retract_Count 0x0032   253   253   000    Old_age   Always       -       0

193 Load_Cycle_Count        0x0032   253   253   000    Old_age   Always       -       0

194 Temperature_Celsius     0x0032   044   253   000    Old_age   Always       -       49

195 Hardware_ECC_Recovered  0x000a   253   252   000    Old_age   Always       -       13484

196 Reallocated_Event_Count 0x0008   124   124   000    Old_age   Offline      -       129

197 Current_Pending_Sector  0x0008   253   241   000    Old_age   Offline      -       0

198 Offline_Uncorrectable   0x0008   124   124   000    Old_age   Offline      -       129

199 UDMA_CRC_Error_Count    0x0008   199   199   000    Old_age   Offline      -       0

200 Multi_Zone_Error_Rate   0x000a   253   252   000    Old_age   Always       -       0

201 Soft_Read_Error_Rate    0x000a   253   252   000    Old_age   Always       -       0

202 Data_Address_Mark_Errs  0x000a   253   247   000    Old_age   Always       -       0

203 Run_Out_Cancel          0x000b   253   252   180    Pre-fail  Always       -       2

204 Soft_ECC_Correction     0x000a   253   252   000    Old_age   Always       -       0

205 Thermal_Asperity_Rate   0x000a   253   252   000    Old_age   Always       -       0

207 Spin_High_Current       0x002a   253   252   000    Old_age   Always       -       0

208 Spin_Buzz               0x002a   253   252   000    Old_age   Always       -       0

209 Offline_Seek_Performnce 0x0024   239   239   000    Old_age   Offline      -       168

210 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

211 Unknown_Attribute       0x0032   253   252   000    Old_age   Always       -       0

212 Unknown_Attribute       0x0032   253   253   000    Old_age   Always       -       0
```

So everything appears to be fine, as Current_Pending_Sector is 0.

Regards

----------

