target

  • echo 'blacklist qla2xxx' > /etc/modprobe.d/blacklist-qla2xxx.conf
  • rmmod qla2xxx
  • echo 'options qla2xxx_scst qlini_mode="disabled"' > /etc/modprobe.d/qla2xxx_scst.conf
  • echo 'deb http://repository.atlas.local/debian jessie-backports contrib non-free main' > /etc/apt/sources.list.d/jessie.list
  • apt update
  • apt install firmware-qlogic subversion fakeroot kernel-wedge build-essential makedumpfile kernel-package lsscsi
  • apt install -t jessie-backports linux-headers-4.9.0-0.bpo.3-amd64
  • apt install -t jessie-backports zfs-dkms
  • apt install -t jessie-backports linux-image-4.9.0-0.bpo.3-amd64
  • update-grub
  • reboot (power off, power on)
  • route add default gw 10.20.30.100
  • svn co https://svn.code.sf.net/p/scst/svn/trunk scst
  • cd scst
  • make clean
  • make 2release
  • BUILD_2X_MODULE=y CONFIG_SCSI_QLA_FC=y CONFIG_SCSI_QLA2XXX_TARGET=y make all install
  • depmod
  • for m in scst qla2xxx_scst qla2x00tgt scst_vdisk scst_user scst_disk; do modprobe $m; done
  • cd scstadmin
  • make
  • make install
  • write into /etc/scst.conf
HANDLER vdisk_fileio {
        DEVICE zfs {
                filename /dev/zvol/data/volume
        }
}

TARGET_DRIVER copy_manager {
        TARGET copy_manager_tgt {
                LUN 0 zfs
        }
}

TARGET_DRIVER qla2x00t {
        TARGET 21:00:00:24:ff:2b:b1:7c {
                HW_TARGET
                enabled 1
                GROUP drives {
                        LUN 0 zfs

                        INITIATOR 21:00:00:1b:32:93:d6:0f
                }
        }
        TARGET 21:00:00:24:ff:2b:b1:7d {
                HW_TARGET
                enabled 0
        }
}

zfs

  • zpool create -o ashift=12 data raidz2 sdc sdd sde sdf sdg sdh sdi sdj sdk sdl sdm sdn cache /dev/sdb1 log /dev/sdb2
  • zfs create -s -V 47T data/volume
  • zfs set compression=on data/volume

export volume

  • scstadmin -config /etc/scst.conf

zoning on FC switch

  • alicreate "SCR04","21:00:00:24:ff:2b:b1:7c"
  • zonecreate "MD4_SCR04","MD4;SCR04"
  • cfgadd "SL3000_DDN_6780","MD4_SCR04"
  • cfgsave
  • cfgenable "SL3000_DDN_6780"

on initiator

  • edit /etc/driver/drv/scsi_vhci.conf
and add the line scsi-vhci-failover-override = "SCST_FIOfc", "f_sym";
  • reboot the mds
  • create labels and partitions, e.g. with zpool create foo device; zpool destroy foo
  • sammkfs -a 64 Linux
  • mkdir /test
  • mount -F samfs Linux /test

monitoring throughput

It could by rubinized with reading information automatically.

#!/bin/bash


IN1=21:00:00:24:ff:37:a5:1c
IN2=21:00:00:24:ff:37:a5:1d

while [ TRUE ]
do
        t1=$( date +"%s.%N" )
        wc11=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN1/write_io_count_kb )
   rc11=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN1/read_io_count_kb )
        wc12=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN2/write_io_count_kb )
   rc12=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN2/read_io_count_kb )
        sleep 2
        t2=$( date +"%s.%N" )
        wc21=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN1/write_io_count_kb )
   rc21=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN1/read_io_count_kb )
        wc22=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN2/write_io_count_kb )
   rc22=$( cat /sys/kernel/scst_tgt/targets/qla2x00t/$IN2/read_io_count_kb )
        rw1=$( echo "scale=2;($wc21-$wc11)/($t2-$t1)/1000"| bc )
        rr1=$( echo "scale=2;($rc21-$rc11)/($t2-$t1)/1000"| bc )
        rw2=$( echo "scale=2;($wc22-$wc12)/($t2-$t1)/1000"| bc )
        rr2=$( echo "scale=2;($rc22-$rc12)/($t2-$t1)/1000"| bc )
       echo "write: \e[91m$rw1\e[39m Mb/s\tread: \e[92m$rr1\e[39m Mb/s write: \e[91m$rw2\e[39m Mb/s\tread: \e[92m$rr2\e[39m Mb/s"
done

Config file for multipath

HANDLER vdisk_fileio {
        DEVICE zfs {
                filename /dev/zvol/data/volume
        }
}

TARGET_DRIVER copy_manager {
        TARGET copy_manager_tgt {
                LUN 0 zfs
        }
}

TARGET_DRIVER qla2x00t {
        TARGET 21:00:00:24:ff:2b:b1:7c {
                HW_TARGET
                enabled 1
                GROUP drives {
                        LUN 0 zfs
                        INITIATOR 21:00:00:1b:32:93:d6:0f
                }
        }
        TARGET 21:00:00:24:ff:2b:b1:7d {
                HW_TARGET
                enabled 1
                GROUP drives {
                        LUN 0 zfs
                        INITIATOR 21:00:00:1b:32:92:8f:af
                }
        }
}
  • scstadmin -config /etc/scst.conf shouhd now make the WWN 21:00:00:24:ff:2b:b1:7d visible on the switch.

additional zoning on the other switch (if it is on the same switch use something else but SCR04)

  • alicreate "SCR04","21:00:00:24:ff:2b:b1:7d"
  • zonecreate "MD3_SCR04","MD3;SCR04"
  • cfgadd "SL3000_DDN_6780","MD3_SCR04"
  • cfgsave
  • cfgenable "SL3000_DDN_6780"

-- HenningFehrmann - 29 May 2017

Tests

Reboot of the Linux disk array

  • the FS is still accessible, probably as long as writings can be buffered and the content is in the cache
  • the logs show
 offlining LUN=0x2 (trace=0x0), target=0x21a00 (trace=0x2800004)
Dec 12 08:27:01 mds-test scsi: [ID 243001 kern.info] /pci@0,0/pci10de,377@a/pci1077,15d@0/fp@0,0 (fcp1):
Dec 12 08:27:01 mds-test        offlining LUN=0x1 (trace=0x0), target=0x21a00 (trace=0x2800004)
Dec 12 08:27:01 mds-test scsi: [ID 243001 kern.info] /pci@0,0/pci10de,377@a/pci1077,15d@0/fp@0,0 (fcp1):
Dec 12 08:27:01 mds-test        offlining LUN=0x0 (trace=0x0), target=0x21a00 (trace=0x2800004)
Dec 12 08:27:01 mds-test genunix: [ID 483743 kern.info] /scsi_vhci/disk@g6237316136666137 (sd27) multipath status: failed: path 4 fp1/disk@w21000024ff26b76b,2 is offline
Dec 12 08:27:01 mds-test genunix: [ID 483743 kern.info] /scsi_vhci/disk@g3833666463373365 (sd25) multipath status: failed: path 5 fp1/disk@w21000024ff26b76b,1 is offline
Dec 12 08:27:01 mds-test genunix: [ID 483743 kern.info] /scsi_vhci/disk@g3930356635663439 (sd26) multipath status: failed: path 6 fp1/disk@w21000024ff26b76b,0 is offline
Dec 12 08:27:09 mds-test scsi: [ID 107833 kern.warning] WARNING: /scsi_vhci/disk@g3930356635663439 (sd26):
Dec 12 08:27:09 mds-test        Command failed to complete...Device is gone


Dec 12 08:29:41 mds-test fctl: [ID 517869 kern.warning] WARNING: fp(0)::N_x Port with D_ID=11a00, PWWN=21000024ff26b76a reappeared in fabric
Dec 12 08:29:42 mds-test fctl: [ID 517869 kern.warning] WARNING: fp(1)::N_x Port with D_ID=21a00, PWWN=21000024ff26b76b reappeared in fabric

exchange the QLogic card

  • configure the zoning on the switch and modify the /etc/scst.conf on the target side
  • the mds recognizes the new paths and lists the old ones as unavailable. Don't know yet, how to get rid of the unavailable paths.

Topic revision: r12 - 07 Mar 2018, HenningFehrmann
This site is powered by FoswikiCopyright © by the contributing authors. All material on this collaboration platform is the property of the contributing authors.
Ideas, requests, problems regarding Foswiki? Send feedback