1 registered members (AbrahamR),
717
guests, and 4
spiders. |
Key:
Admin,
Global Mod,
Mod
|
|
|
Re: Bulk-parse CSV files from whole directory
[Re: jack_the_whack]
#486387
08/09/22 09:22
08/09/22 09:22
|
Joined: Aug 2022
Posts: 3
jack_the_whack
OP
Guest
|
OP
Guest
Joined: Aug 2022
Posts: 3
|
I think this has worked.... Not sure where I was going wrong before. for anyone else who is trying this. Once you have the csv files, put them into the history directory. (13000 gets a bit much but it still works). And then use the following:
// Strategy template ///////////////////////
// MKTS Daily line format: Date,Open,High,Low,Close,Volume
// 02/28/2020,108.4,110,107.475,107.575,44239
// string Format = "+%m/%d/%Y,f3,f1,f2,f4,f6";
string Format = "+%Y-%m-%d,f3,f1,f2,f4,f6";
function writeTheFile(string InName){
if(!InName) return quit("No file");
// apparently resets the file read thing
dataNew(1,0,0);
string OutName = strx(InName,".csv",".t6");
dataParse(1,Format,InName);
printf("writeTheFile: %s \n", OutName);
dataSave(1,OutName);
}
function run() {
set(LOGFILE);
// you should change this to your history location folder
string Name = file_next("D:\\asset-data\\norgate-as-csv\\*.csv");
printf("the first file name is: %s \n", Name);
bool breakLoop = false;
while(!breakLoop){
Name = file_next(0);
printf("next file name is: %s \n", Name);
writeTheFile(Name);
if(Name==NULL){
printf("last value: %s \n", Name);
breakLoop = true;
}
}
printf("success \n");
return quit();
}
Last edited by jack_the_whack; 08/09/22 09:23. Reason: Formatting
|
|
|
Re: Bulk-parse CSV files from whole directory
[Re: jack_the_whack]
#486654
09/13/22 17:35
09/13/22 17:35
|
Joined: Aug 2022
Posts: 65
alun
Junior Member
|
Junior Member
Joined: Aug 2022
Posts: 65
|
Hi jack_the_whack! Thanks for posting this. Some thoughts: 1) f5, f6 (fVal, fVol t6 data) cannot be used in a free version of Zorro, only Zorro S 2) you can use main() instead of run() for the entry point if your function runs once 3) I think you should check if(Name==NULL) before you use name 4) it might work if you just write `while(Name = file_next(0))` so the loop is ended when it returns NULL I have a Java t6 generation code as well - https://gist.github.com/alun/79c4a633c72ea91ec338a7d66c5ab496This part in particular https://gist.github.com/alun/79c4a633c72ea91ec338a7d66c5ab496#file-gent6history-java-L174I wrote it to export data from DukasCopy broker. Maybe it could be useful Also - looks like your format should be
// 02/28/2020,108.4,110,107.475,107.575,44239
string Format = "+%d/%m/%Y,f3,f1,f2,f4,f6";
Last edited by alun; 09/13/22 17:40.
|
|
|
Re: Bulk-parse CSV files from whole directory
[Re: jack_the_whack]
#486884
11/07/22 10:09
11/07/22 10:09
|
Joined: Oct 2022
Posts: 5
2cents
Newbie
|
Newbie
Joined: Oct 2022
Posts: 5
|
Fixed and rewritten the script, now it works properly. It is just an enhancement of the original CSVtoHistory. Please remember, to open the split files outside the Zorro/History folder you need to edit the Chart script. ////////////////////////////////////////////////
// Bulk convert price history from .csv to .t6
// The Format string determines the CSV format (see examples)
////////////////////////////////////////////////
//#define SPLIT_YEARS // split into separate years
//#define FIX_ZONE -1 // add a time zone difference, f.i. for converting CST -> EST
/* T6 Target format:
DATE time;
float fHigh, fLow; // f1,f2
float fOpen, fClose; // f3,f4
float fVal, fVol; // f5,f6
*/
// HISTDATA line format: "20100103 170000;1.430100;1.430400;1.430100;1.430400;0"
//string Format = "+%Y%m%d %H%M%S;f3;f1;f2;f4";
// YAHOO line format "2015-05-29,43.45,43.59,42.81,42.94,10901500,42.94"
//string Format = "%Y-%m-%d,f3,f1,f2,f4,f6,f5"; // unadjusted
// TRADESTATION line format "06/30/2016,17:00:00,2086.50,2086.50,2086.50,2086.50,319,0"
//string Format = "+%m/%d/%Y,%H:%M:%S,f3,f1,f2,f4,f6,f5";
// STK line format "12/23/2016,2300.00,SPY, 225.63, 225.68, 225.72, 225.62,1148991"
//string Format = "+%m/%d/%Y,%H%M,,f3,f4,f1,f2,f6";
// CHRIS_ICE line format: Date,Open,High,Low,Settle,Change,Wave,Volume,...
// 2020-04-08,10.34,10.46,10.22,10.37,-0.01,10.32,54520.0,268936.0,4008.0,50.0,500.0
//string Format = "%Y-%m-%d,f3,f1,f2,f4,,,f6";
// MKTS Daily line format: Date,Open,High,Low,Close,Volume
// 02/28/2020,108.4,110,107.475,107.575,44239
string Format = "+%m/%d/%Y,f3,f1,f2,f4,f6";
// you should change this to your history location folder
//string Name = file_next("D:\\asset-data\\norgate-as-csv\\");
//string Name = file_next("C:\\QuantDataManager\\export\\");
string historyFolder = "D:\\Put\\Here\\the Path\\of your\\Personal\\History Folder\\";
string Csv = "*.csv";
char Path[0];
int a;
function writeTheFile(string InName) {
if (!InName) return quit("No file");
printf("Filename control: %s \n", InName);
int Records = dataParse(1, Format, InName);
printf("\n%d lines read", Records);
a++;
#ifdef FIX_ZONE
int i;
for (i = 0; i < Records; i++)
dataSet(1, i, 0, dataVar(1, i, 0) + FIX_ZONE / 24.);
#endif
#ifndef SPLIT_YEARS
string OutName = strx(InName, ".csv", ".t6");
if (Records) dataSave(1, OutName);
printf("\n%s", OutName);
#else
int i, Start = 0, Year, LastYear = 0;
for (i = 0; i < Records; i++) {
Year = ymd(dataVar(1, i, 0)) / 10000;
if (!LastYear) LastYear = Year;
if (i == Records - 1) { // end of file
LastYear = Year;
Year = 0;
i++;
}
if (Year != LastYear) {
string OutName = strf("%s_%4i.t6", strxc(InName, '.', 0), LastYear);
printf("\n%s", OutName);
dataSave(1, OutName, Start, i - Start);
Start = i;
LastYear = Year;
}
}
#endif
dataNew(1, 0, 0);
}
function run() {
set(LOGFILE);
strcpy(Path, historyFolder);
string Name = file_next(strcat(Path, Csv));
strcpy(Path, historyFolder);
strcat(Path, Name);
writeTheFile(Path);
bool breakLoop = false;
while (!breakLoop) {
Name = file_next(0);
if (Name == NULL) {
break;
}
strcpy(Path, historyFolder);
strcat(Path, Name);
writeTheFile(Path);
}
printf("success, %d files converted \n", a);
return quit();
} Any suggestions or adjustments are welcome. There might be a better way to handle strings, it's my first time with C. I come from a world where const a = "This is a string"
let b = a + "and this" + "is a string too"
Don't poke the bear
|
|
|
|