Plateforme Level Extreme
Abonnement
Profil corporatif
Produits & Services
Support
Légal
English
Crc32
Message
De
16/08/2011 07:20:25
 
Information générale
Forum:
ASP.NET
Catégorie:
Autre
Titre:
Crc32
Versions des environnements
Environment:
C# 4.0
Divers
Thread ID:
01521077
Message ID:
01521077
Vues:
183
J'aime (1)
Michel,

Have been busy looking at the algorithm, and I have found that

(1) There are different crc32, pkzip uses the crc32 IEEE, intel uses CRC32 Castagnoli
There is also Koopman and Q
See http://en.wikipedia.org/wiki/Cyclic_redundancy_check#cite_note-koop02-10 Commonly used and standardized CRCs

The difference is the polynomial they start with - see right side of the table: polynomial and polynomial reversed,.

I think the polynomial reversed is used for little endian systems ( intel for sure)

(2) Timing
The code you have posted is the Sarwate algorithm. It works by creating a lookup table of the remainder of all possible 8 bit numbers divided by the polynomial.
The division is a long division modulo-2

I have run that on a file of half a GB. That takes 3.4 seconds on my computer

(3) Slicing-by-8 algorithm
I have found this paper: http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf

The code on the right of page 7 shows how to compute the crc with 8 precalulated tables.

Been busy with precalulating those

Once implemented, the Slicing-by-8 calculates the crc in about 2/3 of the time

I'm including here

(1) an example of how to calculate the crc

(2) The base class

(3) The class for the crc32 IEEE

(4) The class for the CRC32C aka Castagnoli

Addition of the others (Koopman and Q) should be fairly simple as only the polynomial is different

(1) an example of how to calculate the crc
		static void Test()
		{
			//0xD97531B2
			string filename = @"M:\VM\Machines\Vfp9_Sp1\Vfp9_Sp1_D.vmdk";

			Int32 crc_sliced;

			using (Stream s = File.OpenRead(filename))
			{
				var vv = new CRC_IEEE();
				crc_sliced = vv.GetCRC32(s);
				Console.WriteLine("{0} sliced", crc_sliced.ToString("X8"));
			}
}
(2) The base class
	/*
	 * http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf
	 * Using ' Slicing-by-8' algorithm
	 */
	public abstract class CRC_Base
	{

		private const UInt32 CRC32InitValue = 0xffffffff;
		//______________________________________________________________________
		protected abstract UInt32[] Polynomial32definition { get; }
		//______________________________________________________________________
		protected  UInt32[] Table32_32;
		protected  UInt32[] Table32_40;
		protected  UInt32[] Table32_48;
		protected  UInt32[] Table32_56;
		protected  UInt32[] Table32_64;
		protected  UInt32[] Table32_72;
		protected  UInt32[] Table32_80;
		protected  UInt32[] Table32_88;
		//______________________________________________________________________
		protected CRC_Base()
			: base()
		{
			MakeTables32();
		}
		//______________________________________________________________________
		private void MakeTables32()
		{
			UInt32 poly32Reversed = MakeReversePolynomial(Polynomial32definition);
			Table32_32 = MakeTable(poly32Reversed, 0);
			Table32_40 = MakeTable(poly32Reversed, 8);
			Table32_48 = MakeTable(poly32Reversed, 16);
			Table32_56 = MakeTable(poly32Reversed, 24);
			Table32_64 = MakeTable(poly32Reversed, 32);
			Table32_72 = MakeTable(poly32Reversed, 40);
			Table32_80 = MakeTable(poly32Reversed, 48);
			Table32_88 = MakeTable(poly32Reversed, 56);
		}
		//______________________________________________________________________
		private static UInt32 MakeReversePolynomial(UInt32[] powers)
		{
			UInt32 polynomial = 0;
			for (int i = 0; i <= powers.GetUpperBound(0); i++)
				polynomial |= 1u << (int)(31u - powers[i]);

			//Console.WriteLine("poly {0}", polynomial.ToString("X8"));
			return polynomial;
		}
		//______________________________________________________________________
		// http://en.wikipedia.org/wiki/Computation_of_CRC
		// Code fragment 5: Shift register based division, LSB first
		private static UInt32[] MakeTable(UInt32 polynomial, int offset)
		{
			UInt32[] table = new UInt32[256];

			UInt64 c, x, rem;
			for (int i = table.GetLowerBound(0); i <= table.GetUpperBound(0); i++)
			{
				c = ((UInt64)i) << offset;
				rem = 0;

				for (int part = 8; --part >= 0; )
				{
					x = c >> 56;
					c <<= 8;

					rem ^= x;
					for (int j = 8; --j >= 0; )
						rem = (rem & (0x01u)) != 0 ? polynomial ^ (rem >> 1) : (rem >> 1);

				}

				table[i] = (UInt32)rem;
			}
			//Console.WriteLine("table offset {0} {1}", offset, table[255].ToString("X8"));
			return table;
		}
		//______________________________________________________________________
		public Int32 GetCRC32( Stream stream)
		{
			UInt32 crc32 = CRC32InitValue;

			int bufSize =  4096 ;

			byte[] buf = new byte[bufSize];
			int count;

			WarmTables();

			while ((count = stream.Read(buf, 0, bufSize)) > 0)
			{
				crc32 = HashBlock(crc32, buf, count);
			}

			crc32 = ~crc32;

			return unchecked((Int32)crc32);

		}
		//______________________________________________________________________
		private void WarmTables()
		{
			UInt32 touch;
			for (int i = 0; i < 256; i++)
			{
				touch = Table32_32[i];
				touch = Table32_40[i];
				touch = Table32_48[i];
				touch = Table32_56[i];
				touch = Table32_64[i];
				touch = Table32_72[i];
				touch = Table32_80[i];
				touch = Table32_88[i];
			}
		}
		//______________________________________________________________________
		private UInt32 HashBlock(UInt32 crc32, byte[] buf, int count)
		{
			int headCycles = count >> 3;
			int tailCycles = count & 0x07;

			int offset = 0;
			// head
			for (int i = headCycles; --i >= 0; offset +=8)
			{
				crc32 ^= BitConverter.ToUInt32(buf, offset);

				crc32 = Table32_88[crc32 & 0xFF] 
						^ Table32_80[(crc32 >> 8) & 0xFF]
						^ Table32_72[(crc32 >> 16) & 0xFF]
						^ Table32_64[(crc32 >> 24) & 0xFF]
						^ Table32_56[buf[offset + 4]]
						^ Table32_48[buf[offset + 5]]
						^ Table32_40[buf[offset + 6]]
						^ Table32_32[buf[offset + 7]];

			}

			// tail
			if( tailCycles != 0 )
				for (int i = tailCycles; --i >= 0; )
					crc32 = Table32_32[(crc32 ^ (UInt32)buf[offset++]) & 0xFF] ^ (crc32 >> 8);

			return crc32;

		}
		//______________________________________________________________________
	}
(3) The class for the crc32 IEEE
	// IEE implementation
	// ISO 3309, ITU-T V.42, Ethernet, SATA, MPEG-2, Gzip, PKZIP, POSIX cksum, PNG
	public class CRC_IEEE : CRC_Base
	{
		//______________________________________________________________________
		protected override UInt32[] Polynomial32definition
		{
			get
			{
				return new UInt32[] { 26, 23, 22, 16, 12, 11, 10, 8, 7, 5, 4, 2, 1, 0 };
			}
 			
		}
		//______________________________________________________________________
		public CRC_IEEE()
			: base()
		{
		}
	}
(4) The class for the CRC32C aka Castagnoli
	// Castagnoli
	// iSCSI & SCTP, G.hn payload, SSE4.2
	public class CRC_C : CRC_Base
	{
		//______________________________________________________________________
		protected override UInt32[] Polynomial32definition
		{
			get
			{
				return new UInt32[] { 28, 27, 26, 25, 23, 22, 20, 19, 18, 14, 13, 11, 10, 9, 8, 6, 0 };
			}
 			
		}
		//______________________________________________________________________
		public CRC_C()
			: base()
		{
		}
	}
Gregory
Suivant
Répondre
Fil
Voir

Click here to load this message in the networking platform