Hi Thomas,

thank you for the MD5 checksum idea. Using one of the MD5 FLL functions the records in the resulting Table2.dbf are still identical to my previous pure VFP solution. Craig Boyd's implementation is usually 0.4 seconds faster than the one from Ed Leafe's site.

Now we're below 10 seconds. :-)

Markus

Just in case someone lost track this is the code to beat:

*===========================================================
* Tested with 500,000 records in Vfp5/6/7/8/9 and done in
* less than 10 seconds. :-)
* You can download test data from here:
* http://indot.net/dotnetprocontest07/testdata.zip
* or generate your own test data using
* http://indot.net/dotnetprocontest07/datagenerator.zip
* You can download VfpEncryption.fll from here:
* http://www.sweetpotatosoftware.com/files/vfpencryption.zip
*===========================================================
*--------------------------------------------
* Constants.
*--------------------------------------------
#DEFINE _TIMING_ .T.
#DEFINE _FOLDER_ "C:\Daten\"

*--------------------------------------------
* Setup.
*--------------------------------------------
#IF _TIMING_
  CLEAR ALL
  LOCAL lnSec0, lnSec1, lnSec2, lnSec3
  lnSec0 = SECONDS()
#ENDIF
SET COLLATE TO "MACHINE"
SET TALK OFF
SET NOTIFY OFF
SET NOTIFY CURSOR OFF
SET LIBRARY TO _FOLDER_ + "VfpEncryption.fll"
LOCAL lnFHandle

*--------------------------------------------
* Create a table and import CSV data.
*--------------------------------------------
ERASE _FOLDER_ + "Table*.*"
ERASE _FOLDER_ + "Adressen2.txt"
CREATE TABLE _FOLDER_ + "Table1" FREE ( ;
  Vorname C(11), ;
  Name C(11), ;
  Strasse C(21), ;
  HausNr C(3), ;
  PLZ C(7), ;
  Ort C(18), ;
  eMail C(38) )
APPEND FROM _FOLDER_ + "Adressen.txt" DELIMITED
REPLACE ALL ;
  Name WITH LTRIM( Name ), ;
  Strasse WITH LTRIM( Strasse ), ;
  HausNr WITH LTRIM( HausNr ), ;
  PLZ WITH LTRIM( PLZ ), ;
  Ort WITH LTRIM( Ort ), ;
  eMail WITH LTRIM( eMail )

lnSec1 = SECONDS()

*--------------------------------------------
* Copy only the unique records to a new table
* using a different table structure.
*--------------------------------------------
SET UNIQUE ON
INDEX ON Hash( Vorname + Name + Strasse + HausNr + PLZ + Ort + eMail, 5 ) TO _FOLDER_ + "Table1.idx" COMPACT
CREATE TABLE _FOLDER_ + "Table2" FREE ( ;
  Name C(22), ;
  Strasse C(23), ;
  Ort C(24), ;
  eMail C(37) )
SELECT Table1
SCAN
  INSERT INTO "Table2" VALUES ( ;
    RTRIM( Table1.Vorname ) +" "+ Table1.Name, ;
    RTRIM( Table1.Strasse ) +" "+ Table1.HausNr, ;
    Table1.PLZ + Table1.Ort, ;
    Table1.eMail )
ENDSCAN
USE IN Table1

lnSec2 = SECONDS()

*--------------------------------------------
* Export to CSV. The required CSV format is
* different from VFP's default (same as the
* input file).
*--------------------------------------------
lnFHandle = FCREATE( _FOLDER_ + "Adressen2.txt", 0 )
*ASSERT NOT m.lnFHandle == -1
SELECT Table2
SCAN
  FPUTS( m.lnFHandle, ;
    STUFF( RTRIM( Name ), AT( " ", Name ), 1, ", " ) +", "+ ;
    STUFF( RTRIM( Strasse ), RAT( " ", RTRIM( Strasse ) ), 1, ", " ) +", "+ ;
    STUFF( RTRIM( Ort ), AT( " ", Ort ), 1, ", " ) +", "+ ;
    RTRIM( eMail ) )
ENDSCAN
FCLOSE( m.lnFHandle )

*--------------------------------------------
* Done.
*--------------------------------------------
#IF _TIMING_
  lnSec3 = SECONDS()
  ACTIVATE SCREEN
  ?
  ? "overall:", m.lnSec3 - m.lnSec0
  ? "part 1 :", m.lnSec1 - m.lnSec0
  ? "part 2 :", m.lnSec2 - m.lnSec1
  ? "part 3 :", m.lnSec3 - m.lnSec2
#ENDIF