Posted By: Fiber
CSV to Tick import problem - 08/17/17 09:17
Hi,
I downloaded script CSVtoT6andT1.c from the forum and trying to import tick data from CSV.
After running import, only every second line is imported from CSV file, though Zorro messages states that all lines are processed. What could be a problem?
I downloaded script CSVtoT6andT1.c from the forum and trying to import tick data from CSV.
After running import, only every second line is imported from CSV file, though Zorro messages states that all lines are processed. What could be a problem?
Code:
// convert price history from .csv to .t1 or .t6 #define TRADESTATION //#define HISTDATA //#define YAHOO // convert csv data from R or Yahoo #ifdef TRADESTATION #define TCK T1 string InName = "History\ticks.txt"; // name of the CSV file string OutName = "History\ticks.t1"; #endif #ifdef HISTDATA #define TCK T6 string InName = "History\DAT_ASCII_USDZAR_M1_2015.csv"; // name of the CSV file string OutName = "History\USDZAR_2015hd.t6"; #endif #ifdef YAHOO #define TCK T6 string InName = "History\table.csv"; // name of the CSV file string OutName = "History\Stock.t6"; #endif typedef struct _SYSTEMTIME { WORD wYear; WORD wMonth; WORD wDayOfWeek; WORD wDay; WORD wHour; WORD wMinute; WORD wSecond; WORD wMilliseconds; } SYSTEMTIME; API(SystemTimeToVariantTime,oleaut32) int _stdcall SystemTimeToVariantTime(SYSTEMTIME* lpSystemTime, double* pvtime); DATE ConvertTime(int Year,int Month,int Dom,int Hour,int Minute,int Second) { SYSTEMTIME Time; memset(&Time,0,sizeof(SYSTEMTIME)); if(Year < 50) Year += 2000; else if(Year < 100) year += 1900; Time.wYear = Year; Time.wMonth = Month; Time.wDay = Dom; Time.wHour = Hour; Time.wMinute = Minute; Time.wSecond = Second; DATE vTime; SystemTimeToVariantTime(&Time,&vTime); #ifdef HISTDATA vTime += 5./24.; // add 5 hours for converting EST to UTC #endif return vTime; } string readTick(string content,TCK* tick) { // tokenize a single line char* line = strtok(content,"n"); if(!line) return 0; int Year, Month, Dom, Hour = 0, Minute = 0, Second = 0; #ifdef HISTDATA // line format "20100103 170000;1.430100;1.430400;1.430100;1.430400;0" if(10 != sscanf(line,"%4d%2d%2d %2d%2d%2d;%f;%f;%f;%f;", &Year, &Month, &Dom, &Hour, &Minute, &Second, &tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose)) return 0; tick->fVal = tick->fVol = 0; #endif #ifdef YAHOO // line format "2015-05-29,43.45,43.59,42.81,42.94,10901500,42.94" float fAdj; if(9 != sscanf(line,"%4d-%2d-%2d,%f,%f,%f,%f,%f,%f", &Year, &Month, &Dom, &tick->fOpen, &tick->fHigh, &tick->fLow, &tick->fClose,&tick->fVol,&fAdj)) return 0; if(tick->fClose > 0. && fAdj > 0.) { // adjust for dividends and splits float f = fAdj/tick->fClose; tick->fOpen *= f; tick->fHigh *= f; tick->fLow *= f; tick->fClose = fAdj; tick->fVal = 0; } #endif #ifdef TRADESTATION // line format "06/30/2016,17:00:00,2086.50,2086.50,2086.50,2086.50,319,0" if(7 != sscanf(line,"%2d/%2d/%4d,%2d:%2d:%2d,%f", &Month, &Dom, &Year, &Hour, &Minute, &Second, &tick->fVal)) return 0; #endif // store the time in DATE format tick->time = ConvertTime(Year,Month,Dom,Hour,Minute,Second); // return pointer to next line return line+strlen(line)+1; } function main() { if(!file_length(InName)) { quit("Data file not found!"); return; } string content = file_content(InName); // allocate TICK array int maxticks = strcount(content,0x0a); printf("n%d lines processing...",maxticks); int skipped = 0; TCK* ticks = malloc(maxticks*sizeof(TCK)); #ifdef HISTDATA // read ticks in reverse order TCK* tick = ticks+maxticks; while(content) content = readTick(content,--tick); // store the ticks int size = (int)(ticks+maxticks)-(int)(tick+1); file_write(OutName,tick+1,size); #endif #ifdef YAHOO // skip the header content = strstr(content,"n")+1; // already contains ticks in reverse order TCK* tick = ticks; while(content) { content = readTick(content,tick++); } // store the ticks int size = (int)(tick-1)-(int)(ticks); file_write(OutName,ticks,size); #endif #ifdef TRADESTATION // skip the header content = strstr(content,"n")+1; // read ticks in reverse order TCK* tick = ticks+maxticks; while(content) { content = readTick(content,--tick); if(tick < ticks+maxticks-1) { TCK* prev = tick+1; if(tick->fVal == prev->fVal) { ++tick; // skip unchanged quotes ++skipped; } else if(tick->time <= prev->time) tick->time += 1./(24.*60.*60.*100.); // add 10 ms } } // store the ticks int size = (int)(ticks+maxticks)-(int)(tick+1); file_write(OutName,tick+1,size); #endif free(ticks); printf("n%d ticks written",size/sizeof(TCK)); if(skipped) printf(", %d skipped",skipped); }