Testing different zpool layouts on Thumper

Local testing with iozone

Essentially the tests were made with a single iozone run per file system, zfs compression was always 'on', thus no gzip, just out of the box default compression. All other settings were at defaults. System was freshly patched during the final week of July 2010, thus runing the latest and greatest Solaris 10 possible.

The script and full results are attached to this page, the summary was created with this small snippet

perl -ane '/Create zpool with (.+)$/ && print " $1 "; /3554432\s+64\s+(.+)$/ && print "$1"; /all destroyed, test ran for\s+(\d+)/ && print " $1 \n"' zpool-test.log | perl -ane 's/ +/ |  /g; s/^ //; s/ $//; print' > zpool-test.summary

to make it "Wiki compatible".

The command line used for iozone was always:

iozone -a -g 32G -n 32G -r 64k -f /benchtest/iozone.test

Raw results

Here are the results, and it seems not really a clear winner...

layout	raidlv	slog	cache	write	rewrite	read	reread	ranread	ranwrite	bkwdread	recrewrite	strideread	fwrite	frewrite	fread	freread	tottime[s]
2x21	z	n	0	269608	216117	833924	933366	12115	11802	303189	3040443	261285	584414	199220	708446	769130	6652
2x21	z	n	1	674579	143248	836642	880441	22954	33518	531835	3070482	308736	560959	184637	694594	725164	3431
2x21	z	n	2	752154	112669	839945	888675	21672	32851	556415	3069993	299372	546681	184589	720198	699759	3605
2x21	z	y	0	682160	210177	810442	960895	12060	11562	315042	3080350	231029	570309	200289	703155	782807	6763
2x21	z	y	1	708917	146186	815312	1085549	20132	33431	546307	3058520	290395	625003	191915	700400	705954	3708
2x21	z	y	2	666445	150732	807361	1069024	20196	34873	516193	3047566	295818	649802	189662	701983	701634	3647
2x21	z2	n	0	657948	198450	852493	924455	12237	11763	293217	3059922	257356	547567	197526	706287	755985	6591
2x21	z2	n	1	619909	144617	800687	841699	23369	33812	521294	3095624	282606	545227	183350	731128	712919	3424
2x21	z2	n	2	661120	113736	815624	849116	23560	35327	495612	3049247	283474	545015	184970	747243	727250	3431
2x21	z2	y	0	674475	199333	838750	911701	12200	11740	313942	3042294	222081	541754	199802	701198	808076	6736
2x21	z2	y	1	664393	144789	788386	1031455	20139	33708	514284	3083612	263579	581748	183753	671173	680237	3791
2x21	z2	y	2	649454	116132	788618	1057712	20531	35045	494462	3092608	245838	580222	184805	681760	689565	3798
3x14	z	n	0	669830	201656	887758	1018974	12342	11835	349729	3054856	261821	579085	196587	738641	813833	6496
3x14	z	n	1	648502	195002	851862	1056668	20632	33390	639902	3041572	327993	576529	191780	722328	728806	3497
3x14	z	n	2	698470	146544	872762	901228	23349	35360	627158	3074187	328980	579582	185994	733245	730256	3309
3x14	z	y	0	684789	206707	866087	988420	12285	11810	362200	3070839	243786	561390	198077	734634	924057	6600
3x14	z	y	1	782409	146052	835260	1023328	19924	31528	622758	3045218	281310	619206	188321	709430	732718	3819
3x14	z	y	2	641174	149000	826891	1012346	21041	35506	636376	3075529	304393	622729	188846	728590	730130	3548
3x14	z2	n	0	710079	196818	916081	1060766	12117	11704	343980	3036067	274481	577674	192479	746199	823939	6575
3x14	z2	n	1	654094	191342	883450	927753	21069	33277	562295	3053772	289410	559106	187004	722261	737584	3508
3x14	z2	n	2	645491	112684	870155	917645	23390	35141	567282	3063997	314049	526338	185595	722829	733497	3407
3x14	z2	y	0	641044	204286	860361	1035263	12054	11685	343391	3042102	206110	552159	195053	725222	806096	6809
3x14	z2	y	1	686399	144024	848494	1072330	20251	33826	540144	3044210	279985	565056	184313	723380	745578	3735
3x14	z2	y	2	712560	197887	865145	1042923	19902	34444	580736	3052248	280377	568855	183322	730039	732981	3666
6x7	z	n	0	704805	206876	968444	1164177	13117	12639	523755	3039194	363201	593930	206706	791757	957165	6032
6x7	z	n	1	731526	203740	974814	1011237	22574	34393	751339	3052650	380077	575866	198838	787360	939758	3252
6x7	z	n	2	701663	152123	986120	1010458	24408	36158	802589	3052090	405225	572340	192638	780308	833559	3159
6x7	z	y	0	716048	216861	907426	1123987	13105	12615	528092	3068171	309323	546868	204976	754765	825922	6208
6x7	z	y	1	691660	150230	915600	1177226	21600	34830	777144	3065247	338621	557485	194943	764524	806726	3507
6x7	z	y	2	693487	154859	928596	1183206	21545	36283	799840	3050497	371049	571412	199347	787472	956868	3429
6x7	z2	n	0	710803	203781	976216	1177213	12993	12471	494699	3053801	365847	588738	202011	926324	927958	6097
6x7	z2	n	1	700670	146669	969050	1181715	22065	34871	754126	3071382	374973	580719	192606	797984	832882	3345
6x7	z2	n	2	682202	113940	994450	1026681	25237	36480	728592	3088899	385189	537930	188651	788262	825433	3193
6x7	z2	y	0	710028	207753	926104	1166274	12982	12568	493012	3080757	307541	578196	195998	868590	899046	6214
6x7	z2	y	1	699152	117451	950165	1206815	21347	34896	740671	3064047	340476	548175	193083	778727	836133	3595
6x7	z2	y	2	694019	150973	967478	1199274	21790	35934	756085	3040778	363558	558354	191833	799895	818104	3425
7x6	z	n	0	685100	213501	995604	1150946	13611	13242	611511	3050547	406670	583585	207219	814706	891204	5791
7x6	z	n	1	777704	147166	1025379	1056555	25659	35707	878707	3070400	433326	501953	193047	811916	841740	3098
7x6	z	n	2	755051	151225	1056156	1056703	25290	36995	851685	3048915	446110	555577	199762	830899	832717	3064
7x6	z	y	0	673061	215572	956488	1125354	13827	13023	635500	3063851	333338	618845	208063	819668	926053	5930
7x6	z	y	1	668110	151395	959382	1136449	22980	35319	891897	3070370	426712	605312	203288	820870	851093	3323
7x6	z	y	2	740806	151661	987689	1206430	22437	36788	865889	3064504	417144	611612	201793	830880	961914	3317
7x6	z2	n	0	783489	206956	1029375	1191157	14168	13173	528037	3056851	365756	565990	216774	867145	907606	5718
7x6	z2	n	1	572825	149011	993529	1034676	25753	35751	814351	3086990	413522	578647	199510	815593	949706	3088
7x6	z2	n	2	727631	116331	1019946	1118401	23808	37667	796601	3071785	395749	548667	198837	828496	847227	3209
7x6	z2	y	0	677286	213625	960239	1065828	14017	13406	581856	3068321	313042	651050	214607	916907	919039	5835
7x6	z2	y	1	742914	152230	969695	1207610	22608	36001	819717	3044280	335654	590246	201781	815341	952941	3383
7x6	z2	y	2	701182	153892	980033	1162714	23683	37138	820892	3085917	355057	613649	198680	825201	951895	3255

Some notes:

• The box has 48 slots
• c6t4d0 is our boot device
• 2 slots are occupied by Intel X25 SSDs (32GB) -> mirrored SLOG
• 2 slots are used by Intel M25 (G2) SSDs (160GB) for Z2ARC/cache
• one extra for hot spare drive
• this leaves about 42 usable slots, thus there are only 4 possible zpool configurations:
  • 2x21 - 2 vdevs with 21 drives each, great for capacity, will not help against a full controller loss
  • 3x14 - 3 vdevs with 14 drives each, ok for capcacity will not help against a full controller loss
  • 6x7 - more vdevs usually mean less capacity more speed, only raidz2 will help against full controller loss
  • 7x6 - least capacity, should be fine even with one controller dead

Reduced matrix

This one only allows for "safe" layouts:

layout	raidlv	slog	cache	write	rewrite	read	reread	ranread	ranwrite	bkwdread	recrewrite	strideread	fwrite	frewrite	fread	freread	tottime[s]
6x7	z2	n	0	710803	203781	976216	1177213	12993	12471	494699	3053801	365847	588738	202011	926324	927958	6097
6x7	z2	n	1	700670	146669	969050	1181715	22065	34871	754126	3071382	374973	580719	192606	797984	832882	3345
6x7	z2	n	2	682202	113940	994450	1026681	25237	36480	728592	3088899	385189	537930	188651	788262	825433	3193
6x7	z2	y	0	710028	207753	926104	1166274	12982	12568	493012	3080757	307541	578196	195998	868590	899046	6214
6x7	z2	y	1	699152	117451	950165	1206815	21347	34896	740671	3064047	340476	548175	193083	778727	836133	3595
6x7	z2	y	2	694019	150973	967478	1199274	21790	35934	756085	3040778	363558	558354	191833	799895	818104	3425
7x6	z	n	0	685100	213501	995604	1150946	13611	13242	611511	3050547	406670	583585	207219	814706	891204	5791
7x6	z	n	1	777704	147166	1025379	1056555	25659	35707	878707	3070400	433326	501953	193047	811916	841740	3098
7x6	z	n	2	755051	151225	1056156	1056703	25290	36995	851685	3048915	446110	555577	199762	830899	832717	3064
7x6	z	y	0	673061	215572	956488	1125354	13827	13023	635500	3063851	333338	618845	208063	819668	926053	5930
7x6	z	y	1	668110	151395	959382	1136449	22980	35319	891897	3070370	426712	605312	203288	820870	851093	3323
7x6	z	y	2	740806	151661	987689	1206430	22437	36788	865889	3064504	417144	611612	201793	830880	961914	3317
7x6	z2	n	0	783489	206956	1029375	1191157	14168	13173	528037	3056851	365756	565990	216774	867145	907606	5718
7x6	z2	n	1	572825	149011	993529	1034676	25753	35751	814351	3086990	413522	578647	199510	815593	949706	3088
7x6	z2	n	2	727631	116331	1019946	1118401	23808	37667	796601	3071785	395749	548667	198837	828496	847227	3209
7x6	z2	y	0	677286	213625	960239	1065828	14017	13406	581856	3068321	313042	651050	214607	916907	919039	5835
7x6	z2	y	1	742914	152230	969695	1207610	22608	36001	819717	3044280	335654	590246	201781	815341	952941	3383
7x6	z2	y	2	701182	153892	980033	1162714	23683	37138	820892	3085917	355057	613649	198680	825201	951895	3255

-- CarstenAulbert - 02 Aug 2010