Hi Kevin
>The example you gave me took around 14 times as long.
The important part is the idea to build a program.
With large amount of data (for less than 100 Records
you don't need a dataengine, you can do all that in RAM
without searchoptimization and so on IMHO)
the penalties of Eval() and VFP being a interpreter
can be avoided. On large data sizes Dragan's approach
in a more optimized version (BuildStringTG()), called
via execscript will outperform even Cetin's and Walter's
interesting idea - at least on my machine.
But if you are working with large data sizes, you need to make sure
that VFP's stringhandling doesn't get in the way.
Piping the result into a file is conceptually near
to a .net stringwriter, while vfp's stringhandling
in some points resembles c# strings:
the memory has to be allocated on each assignment.
Hint:
to get clean measurements, I used a ramdisk for
all file output.
Assign a LARGE file in function setup and have fun.
Further optimizations might be reached via a
MemoryStream-like class or .fll, which would assign
a large buffer instead of always new assigning.
Since all looping except for the scan is eliminated,
I doubt that doing the whole thing in c++ will get you the
further speed gains expected from other rewrites in c++.
my 0.02 EUR
thomas
#define C__FILE_DO fputs
#define C__FAST3ONLY .t.
#define C__EXACT_TIMING .f.
PRIVATE pnReadStart, pnReadDone, pnXmlDone, paFields, pnHandle, pcBufFile, pcCR
DIMENSION paFields[1], paPretag[1], paPostTag[1], paData[1]
STORE 0 TO pnReadStart, pnReadDone, pnXmlDone, pnHandle, pcBufFile, ;
paFields, paPretag[1], paPostTag[1], paData[1]
pcCR = CHR(13)
CLEAR
SET SAFETY OFF
SET TALK OFF
ERASE ("Erg.Txt")
#if C__EXACT_TIMING
SET PROCEDURE TO ..\..\_commonTG\prg\timing additive
= queryPerf_Setup()
#endif
? SYS(3050, 1, 256*1024*1024)
? SYS(3050, 2, 256*1024*1024)
FOR m.pnUpperMod = 1 TO 10000 STEP 100
= OneRun()
NEXT
SET SAFETY ON
SET TALK ON
FUNCTION oneRun()
IF !C__FAST3ONLY AND (m.pnUpperMod<500 OR .f.)
= original()
= OrigNoEval()
= dragan()
= dragan2()
endif
IF !C__FAST3ONLY
= Cetin()
= Meester()
endif
= CetinNoEval()
IF !C__FAST3ONLY
= draganLowLevel()
= dragan2_LowLevel()
endif
= Dragan2_LowLevelB2()
= CetinCopy()
= ScriptExec_LowLevel()
FUNCTION original()
=setup(@paFields)
LOCAL lcXML
lcXML = "<TEST>" + m.pcCR
SCAN
lcXml = m.lcXml + "<CLIENT>" + m.pcCR
FOR f = 1 TO ALEN(paFields, 1)
lcXml = lcXml + "<" + paFields(f, 1) + ">" + ;
RTRIM(TRANSFORM(EVALUATE("Test." + paFields(f, 1)))) +;
"</" + paFields(f, 1) + ">" + m.pcCR
ENDFOR
lcXml = lcXml + "</CLIENT>" + m.pcCR
ENDSCAN
lcXML = lcXML + "</TEST>" + m.pcCR
= alldone(@lcXML)
FUNCTION OrigNoEval
=setup(@paFields)
LOCAL lcXML, lnFields, lnRun
lnFields = SetupArrays()
lcXML = "<TEST>" + m.pcCR
SCAN
scatter MEMO to paData
lcXML = m.lcXML + "<CLIENT>" + m.pcCR
FOR lnRun = 1 TO m.lnFields
lcXML = m.lcXML + ;
paPretag[lnRun] + RTRIM(Transform(paData[lnRun])) + paPostTag[lnRun] + m.pcCR
ENDFOR
lcXML = m.lcXML + "</CLIENT>" + m.pcCR
ENDSCAN
lcXML = m.lcXML + "</TEST>" + m.pcCR
= alldone(@m.lcXML)
FUNCTION Cetin
=setup(@paFields)
= FileStrt()
SCAN
C__FILE_DO(m.pnHandle,"<CLIENT>")
FOR f = 1 TO ALEN(paFields, 1)
C__FILE_DO(m.pnHandle,"<" + paFields(m.f, 1) + ">" + ;
RTRIM(TRANSFORM(EVALUATE("Test." + paFields(m.f, 1)))) +;
"</" + paFields(m.f, 1) + ">")
ENDFOR
C__FILE_DO(m.pnHandle,"</CLIENT>")
ENDSCAN
= FileStop()
FUNCTION CetinNoEval
= setUp(@paFields)
LOCAL lnFields
lnFields = SetupArrays()
= FileStrt()
SCAN
C__FILE_DO(m.pnHandle,"<CLIENT>")
scatter MEMO to paData
FOR lnRun = 1 TO m.lnFields
C__FILE_DO(m.pnHandle, paPretag[m.lnRun] + RTRIM(Transform(paData[m.lnRun])) + paPostTag[m.lnRun] )
ENDFOR
C__FILE_DO(m.pnHandle,"</CLIENT>")
ENDSCAN
= FileStop()
FUNCTION Meester
=setup(@paFields)
cString = "<CLIENT>"
FOR nT = 1 TO ALEN(paFields,1)
IF INLIST(paFields[nT, 2], "C", "M")
cString = cString + "<"+paFields(nT,1)+">%%RTRIM("+paFields(nT,1)+")%%</"+paFields(nT,1)+">%%CHR(13)%%"
else
cString = cString + "<"+paFields(nT,1)+">%%RTRIM(TRANSFORM("+paFields(nT,1)+"))%%</"+paFields(nT,1)+">%%m.pcCR%%"
endif
ENDFOR
cString = cString + "</CLIENT>%%CHR(13)Chr(10)%%"
= STRTOFILE(cString, "MEs.txt")
SET TEXTMERGE DELIMITERS TO "%%","%%"
SET TEXTMERGE TO cXml
SET TEXTMERGE on
\\<TEST>
SCAN
TEXTMERGE(cString,.f.)
ENDSCAN
\\</TEST>
SET TEXTMERGE OFF
SET TEXTMERGE TO
SET CONSOLE ON
cXml = FILETOSTR("cxml.txt")
ERASE ("cxml.txt")
= alldone(@m.cXML)
FUNCTION Dragan
=setup(@paFields)
StrToFile(BldStringDragan(), "runner.prg")
Compile runner.prg
LOCAL lcBigXML
lcBigXml="<TEST>" + m.pcCR
Scan
lcBigXml = m.lcBigXml + runner()
ENDSCAN
lcBigXML = lcBigXML + "</TEST>" + m.pcCR + m.pcCR
= alldone(@m.lcBigXML)
FUNCTION DraganLowLevel
=setup(@paFields)
StrToFile(BldStringDragan(), "runner.prg")
Compile runner.prg
= FileStrt()
Scan
fwrite(m.pnHandle,runner())
ENDSCAN
= FileStop()
FUNCTION Dragan2
=setup(@paFields)
StrToFile("return " + BldStringTG(), "runner.prg")
Compile runner.prg
LOCAL lcBigXML
lcBigXml="<TEST>" + m.pcCR
Scan
lcBigXml = m.lcBigXml + runner()
ENDSCAN
lcBigXML = lcBigXML + "</TEST>" + m.pcCR + m.pcCR
= alldone(@m.lcBigXML)
FUNCTION Dragan2_LowLevel
=setup(@paFields)
StrToFile("return " + BldStringTG(), "runner.prg")
Compile runner.prg
= FileStrt()
Scan
C__FILE_DO(m.pnHandle,runner())
ENDSCAN
= FileStop()
FUNCTION Dragan2_LowLevelB2
=setup(@paFields)
StrToFile(ScanString(), "runner.prg")
Compile runner.prg
= FileStrt()
= Runner()
= FileStop()
FUNCTION scanString()
RETURN "Scan" + CHR(13) + CHR(10) + ;
[C__FILE_DO]+"(m.pnHandle, " + BldStringTG() + ")" + CHR(13) + CHR(10) + ;
"EndScan"
FUNCTION ScriptExec_LowLevel()
=setup(@paFields)
= FileStrt()
lcExec = ScanString()
STRTOFILE(lcExec, "Exec.txt")
= EXECSCRIPT(m.lcExec)
= FileStop()
FUNCTION setup(taFields)
SET DECIMALS TO 3
Close Databases all
pnReadStart = getTime_ms()
IF .f.
USE (".Dbf") alias client
ELSE
USE ("..\Audi.Dbf") alias client
endif
SELECT RECNO() as Recnum, * FROM Client INTO CURSOR TEST NOFILTER ;
WHERE BETWEEN(MOD(RECNO(), 10000), 1, m.pnUpperMod)
pnReadDone = getTime_ms()
=AFIELDS(taFields, "Test")
FUNCTION SetupArrays
= SYS(1104)
lnFields = FCOUNT()
dimension paPretag[lnFields], paPostTag[lnFields], paData[lnFields]
for lnRun = 1 to m.lnFields
paPretag[lnRun] = "<" + field(m.lnRun) + ">"
paPostTag[lnRun] = "</" + field(m.lnRun) + ">"
next
RETURN m.lnFields
FUNCTION alldone(tcXml)
pnXmlDone = getTime_ms()
LOCAL lcCaller, lnWorktime
lnWorktime = m.pnXmlDone - m.pnReadDone
lcCaller = PROGRAM(PROGRAM(-1)-1)
IF lcCaller = "FILESTOP"
lcCaller = PROGRAM(PROGRAM(-1)-2)
endif
lcErg = STR(m.pnReadDone - m.pnReadStart, 8) + ;
STR(m.lnWorktime, 8) + ;
STR(LEN(tcXml), 12) + STR(RECCOUNT(), 8) + ;
STR(m.lnWorktime/RECCOUNT(), 8,3) + ;
STR(LEN(tcXml)/m.lnWorktime, 12,3) + ;
" " + m.lcCaller
? lcErg
STRTOFILE(lcErg+CHR(13)+CHR(10), "Erg.Txt", 1)
FUNCTION fileStrt(tnHandle, tcFileTmp)
pcBufFile = filename_tmp()
pnHandle = fcreate(m.pcBufFile)
C__FILE_DO(m.pnHandle,'<TEST>')
FUNCTION fileStop
C__FILE_DO(m.pnHandle,"</TEST>")
Fclose(m.pnHandle)
LOCAL lcBigXML
lcBigXML = FileToStr(m.pcBufFile)
Erase (m.pcBufFile)
= alldone(@m.lcBigXML)
FUNCTION filename_tmp
RETURN "y:" + Sys(2015)+'.tmp'
#if C__EXACT_TIMING
#else
FUNCTION getTime_ms()
return seconds()*1000
#endif
FUNCTION BldStringDragan()
LOCAL lcText
Set Textmerge to memvar lcText noshow
Set Textmerge delimiters to "{{","}}"
Set Textmerge on
\local lcXml
\ lcXML ="<CLIENT>" + m.pcCR
FOR f = 1 TO ALEN(paFields, 1)
\ lcXML = lcXML + "<{{Alltrim(paFields(f, 1))}}>" + ;
\ RTRIM(TRANSFORM(Test.{{Alltrim(paFields(f, 1))}})) +;
\ "</{{Alltrim(paFields(f, 1))}}>" + m.pcCR
ENDFOR
\ lcXML = lcXML + "</CLIENT>" + m.pcCR
\return lcXml
Set Textmerge to
RETURN lctext
FUNCTION BldStringTG()
LOCAL lcText
Set Textmerge to memvar lcText noshow
Set Textmerge delimiters to "{{","}}"
Set Textmerge on
\"<CLIENT>" + m.pcCR + ;
FOR f = 1 TO ALEN(paFields, 1)
lcF = Alltrim(paFields[f, 1])
IF INLIST(paFields[f, 2], "C", "M")
\ "<{{m.lcF}}>" + RTRIM(Test.{{m.lcF}}) + "</{{m.lcF}}>" + m.pcCR + ;
else
\ "<{{m.lcF}}>" + RTRIM(TRANSFORM(Test.{{m.lcF}})) + "</{{m.lcF}}>" + m.pcCR + ;
endif
ENDFOR
IF PROGRAM(PROGRAM(-1)-1) == "DRAGAN2"
\ "</CLIENT>" + m.pcCR
else
\ "</CLIENT>"
endif
Set Textmerge to
RETURN m.lcText
function cetinCopy
LOCAL cChar, cFields, cFile, nReccount, nStartTime, ix
=setup(@paFields)
nReccount = RECCOUNT()
cFile = filename_tmp()
cChar=CHR(13)
cFields = "fb=IIF(RECNO() = 1,'<TEST>"+CHR(13)+"','')+'<CLIENT>',"
FOR ix=1 TO FCOUNT()
IF !TYPE(FIELD(ix))$'GM'
cFields = cFields + "f"+ALLTRIM(STR(ix))+"='<"+FIELD(ix)+">'+" ;
+ IIF(TYPE(FIELD(ix))$'CM', "RTRIM("+FIELD(ix)+")", "RTrim(Transform("+FIELD(ix)+"))")+ ;
+ "+'</"+FIELD(ix)+">',"
ENDIF
ENDFOR
cFields = cFields + "F"+ALLTRIM(STR(ix))+"='</CLIENT>'+IIF(RECNO() = nReccount,CHR(13)+'</TEST>',''))"
SET FIELDS GLOBAL
SET FIELDS TO &cFields
COPY TO (cFile) TYPE DELIMITED WITH "" WITH CHARACTER &cChar
SET FIELDS TO
lcBigXML = FileToStr(m.cFile)
ERASE (cFile)
= alldone(@m.lcBigXML)
RETURN
LOCAL cChar, cFields, cFile, nReccount, nStartTime, ix
SELECT * FROM h:\ids5\dbfs\t_tanklayout INTO CURSOR _tmpXML NOFILTER READWRITE
nReccount = RECCOUNT()
cFile = SYS(2015)+'.tmp'
nStartTime=SECONDS()
cChar=CHR(13)
cFields = "fb=IIF(RECNO() = 1,'<TEST>"+CHR(13)+"','')+'"+CHR(9)+"<CLIENT>',"
FOR ix=1 TO FCOUNT()
IF !TYPE(FIELD(ix))$'GM'
cFields = cFields + "f"+ALLTRIM(STR(ix))+"='"+CHR(9)+CHR(9)+"<"+FIELD(ix)+">'+" + ;
IIF(TYPE(FIELD(ix))$'CM', "TRIM("+FIELD(ix)+")", "Transform("+FIELD(ix)+")")+ ;
"+'</"+FIELD(ix)+">',"
ENDIF
ENDFOR
cFields = cFields + "F"+ALLTRIM(STR(ix))+"='"+chr(9)+"</CLIENT>'+IIF(RECNO() = nReccount,CHR(13)+'</TEST>','')"
SET FIELDS GLOBAL
SET FIELDS TO &cFields
COPY TO (cFile) TYPE DELIMITED WITH "" WITH CHARACTER &cChar
SET FIELDS TO
? SECONDS() - nStartTime
MODIFY FILE (cFile)
ERASE (cFile)
RETURN
