Can't append on CSV

[Andrea Ardemani]

Hello everyone, I'm quite new to the mql5 world, I have a problem. I would like to output the number of parameters of my tests with the tester to a csv file, appending each new string from time to time for each single test that is launched.

The problem is that every time I start a single test (I haven't tried running it on genetic generation yet) the file is overwritten and not queued. What am I doing wrong?

........
string custom_directory = "C:\\Users\\User\\AppData\\Roaming\\MetaQuotes\\Terminal\\Common\\Files\\";
........



.......
void OnDeinit(const int reason)
{
        end_date = TimeToString(TimeCurrent(), TIME_DATE);
        AppendToCSV();
}
.......



........
void AppendToCSV()
{
    // Definisci il nome del file CSV
    string filename = DoubleToString(current_version, 2) + "_" + _Symbol + ".csv";
    string path = custom_directory + filename;
    
    Print("Il file sarà salvato in: ", path);
    
    string oldContent = "";
    if (FileIsExist(filename)) {
        int read_handle = FileOpen(filename, FILE_READ | FILE_CSV | FILE_COMMON);
        if (read_handle != INVALID_HANDLE) {
            while (!FileIsEnding(read_handle)) {
                oldContent += FileReadString(read_handle) + "\n";
            }
            FileClose(read_handle);
        }
    } else {
        // Scrivi le intestazioni se il file non esiste
        oldContent = "Data,Ora,Parità,Timeframe,Deposito Iniziale,Leverage,Ritardo Lag,Periodo di Test Inizio,Periodo di Test Fine,";
        oldContent += "dynamic_initial_lot_size_enabled,dynamic_initial_lot_size_coefficient,initial_lot_size,lot_multiplier,cooling_ratio,";
        oldContent += "profit_percent_to_close,incremental_profit_percent_per_position,rsi_period,rsi_buy_level,rsi_sell_level,";
        oldContent += "initial_pips_loss_quantity,pip_loss_quantity_increase,max_positions,close_all_at_max_positions,withdraw_at,";
        oldContent += "withdraw_amount,enable_limit_max_lot_size,dont_open_above_max_lot_size,max_lot_size,withdraw_total,";
        oldContent += "isFirstCandle,starting_lot_size,lot_size,last_open_price,last_bar_time,pips_loss_quantity,last_position_type,";
        oldContent += "current_lot_multiplier,positions_count,max_drawdown_percent,max_positions_opened,max_lot_size_opened,";
        oldContent += "target_gain_amount,first_withdraw_at\n";
    }
    
    // Raccogli i dati
    string first_withdraw_date = TimeToString(first_withdraw_at, TIME_DATE);
    string date = TimeToString(TimeCurrent(), TIME_DATE);
    string time = TimeToString(TimeCurrent(), TIME_MINUTES);
    string timeframe_str = PeriodToString(PeriodSeconds()/60);

    //Print("Timeframe utilizzato: ", timeframe_str);
    
    string data = StringFormat("%s,%s,%s,%s,%f,%d,%d,%s,%s,", date, time, _Symbol, timeframe_str, AccountInfoDouble(ACCOUNT_BALANCE), AccountInfoInteger(ACCOUNT_LEVERAGE), 0, begin_date, end_date);
    data += StringFormat("%d,%d,%f,%f,%f,", dynamic_initial_lot_size_enabled, dynamic_initial_lot_size_coefficient, initial_lot_size, lot_multiplier, cooling_ratio);
    data += StringFormat("%f,%f,%d,%f,%f,", profit_percent_to_close, incremental_profit_percent_per_position, rsi_period, rsi_buy_level, rsi_sell_level);
    data += StringFormat("%f,%f,%d,%d,%f,", initial_pips_loss_quantity, pip_loss_quantity_increase, max_positions, close_all_at_max_positions, withdraw_at);
    data += StringFormat("%f,%d,%d,%f,%f,", withdraw_amount, enable_limit_max_lot_size, dont_open_above_max_lot_size, max_lot_size, withdraw_total);
    data += StringFormat("%d,%f,%f,%f,%d,", isFirstCandle, starting_lot_size, lot_size, last_open_price, last_bar_time);
    data += StringFormat("%f,%d,%f,%f,%f,", pips_loss_quantity, last_position_type, current_lot_multiplier, positions_count, max_drawdown_percent);
    data += StringFormat("%d,%f,%f", max_positions_opened, max_lot_size_opened, target_gain_amount);
    data += StringFormat(",%s", first_withdraw_date);
    
    // Aggiungi i nuovi dati a oldContent
    oldContent += data + "\n";
    
    // Apri il file in modalità FILE_WRITE e scrivi oldContent
    int write_handle = FileOpen(filename, FILE_WRITE | FILE_CSV | FILE_COMMON);
    if (write_handle != INVALID_HANDLE) {
        FileWriteString(write_handle, oldContent);
        FileClose(write_handle);
    } else {
        Print("Errore nell'apertura del file: ", GetLastError());
    }
}

[Carl Schreiber]

I create the line and append it to a(ny) file:

//use:  addLineToFile("newLine\n", "folder\\name.csv")
bool addLineToFile(const string line, const string fName, const bool shared = true, const int maxPrt=10) {
   ResetLastError();
   static int nErr=0;
   int fH;
   if ( shared )
      fH = FileOpen(fName,FILE_READ|FILE_WRITE|FILE_BIN|FILE_SHARE_READ|FILE_SHARE_WRITE);
   else 
      fH = FileOpen(fName,FILE_READ|FILE_WRITE|FILE_BIN);
   if (fH == INVALID_HANDLE ) { if (nErr<maxPrt) ErrT("File: "+fName+" Open FAILED "+(string(++nErr))); return(false);}
   FileSeek(fH,0,SEEK_END);
   FileWriteString(fH, line, StringLen(line) );
   FileClose(fH);
   if (_LastError<2) return(true);
   if ( nErr < maxPrt ) ErrT("Write "+(string(++nErr))+" to file: "+fName+" went wrong?");
   return(false);
}

A line end (\n) is not added automatically!

[Andrea Ardemani]

Forgive me but I didn't understand your solution, could you explain a little better what you suggest?

[Trinh Dat]

your code always create new file, so you need add FILE_READ

also use FileSeek before write new data

below is code after modified

    // Apri il file in modalità FILE_WRITE e scrivi oldContent
    int write_handle = FileOpen(filename, FILE_READ | FILE_WRITE | FILE_CSV | FILE_COMMON);
    if (write_handle != INVALID_HANDLE) {
	FileSeek(write_handle,0,SEEK_END);
        FileWriteString(write_handle, oldContent);
        FileClose(write_handle);
    } else {
        Print("Errore nell'apertura del file: ", GetLastError());
    }

[Andrea Ardemani]

Thank you! Now it works almost completely, but it keeps repeating the headers, even though it is not a new file! I know what else I'm doing wrong...

[Fernando Carreiro]

Andrea Ardemani #: Thank you! Now it works almost completely, but it keeps repeating the headers, even though it is not a new file! I know what else I'm doing wrong...

Then only write the header when the file is initially created. For example, when the FileSize is zero or when FileIsExist is false, as you did in your original code.

[Fernando Gotopo]

Hi, this article can teach you what you're looking for

https://www.mql5.com/en/articles/2720

[Andrea Ardemani]

Thanks, I solved it! You are always kind.

One thing, with mql5 is it possible to get the current date and time? because if I use TimeLocal() and TimeCurrent() on the tester in the output I see the backtesting date, instead I would like to have the current date and time of when the processing was done.

[Fernando Carreiro]

Andrea Ardemani #: One thing, with mql5 is it possible to get the current date and time? because if I use TimeLocal() and TimeCurrent() on the tester in the output I see the backtesting date, instead I would like to have the current date and time of when the processing was done.

Not possible in normal MQL5, but there are workarounds. Here are two examples:

• Using a DLL to call the Windows API.
• Creating, modifying or accessing a file and obtaining the operation's timestamp.
  

[Andrea Ardemani]

With this code is perfect: https://www.mql5.com/ru/forum/170952/page105#comment_9044250

//+------------------------------------------------------------------+

//| Data e ora Local                                                       |
//+------------------------------------------------------------------+
datetime GetTimeLocal( void )
{ 
  static const bool IsTester = MQLInfoInteger(MQL_TESTER);
  static uint TickCount = 0;
  static datetime InitTimeLocal = 0;
  
  datetime Res = 0;
  
  if (IsTester)
  {
    if (InitTimeLocal)
      Res = InitTimeLocal + (GetTickCount() - TickCount) / 1000;
    else
    {
      int Array[];    
      const string FileName = __FUNCTION__;  
      
      if (FileSave(FileName, Array))
      {
        TickCount = GetTickCount();
        
        Res = InitTimeLocal = (datetime)FileGetInteger(FileName, FILE_MODIFY_DATE);
      }
    }
  }
  else
    Res = TimeLocal();
    
  return(Res);
}

[Andrea Ardemani]

There's something wrong, but I can't figure out why.

On the OnTester I write a line of the csv for each step of the strategy tester ok? However, I don't find all the records. Out of 60 results from the optimizer, I only find 50 on the csv.

So I think it's a writing problem on my disk (even if it seems strange to me, it's a dual xeon 20 core/64gb/nvme, it shouldn't have performance problems) and I try the other solution, which is to write to the csv only lines that meet certain requirements. What happens in this case? He writes me 2/3 of them, then the csv file comes back empty with only the headers. WTF???

So, I'll write the header file immediately by calling it from OnInit, while I write the individual lines with the csv results by calling the dedicated function (AppendToCSV) on the OnTester, with and without the IFs.

void CreateHeadersCSV()
{
    // Definisci il nome del file CSV
    string filename = DoubleToString(current_version, 2) + "_" + _Symbol + ".csv";
    string path = custom_directory + filename;
    string headers = "";
    
    int read_handle = FileOpen(filename, FILE_READ | FILE_CSV | FILE_COMMON);
    
    if (read_handle != INVALID_HANDLE) {
        Print("File csv già presente in: ", path);
        FileClose(read_handle);
        return;
    }
    
    Print("File non presente, il file sarà creato in: ", path);
     
    // Scrivi le intestazioni se il file non esiste
    headers = "Versione,Data,Ora,Parità,Timeframe,Balance_Iniziale,Balance_Finale,Leverage,Periodo di Test Inizio,Periodo di Test Fine,";
    headers += "dynamic_initial_lot_size_enabled,dynamic_initial_lot_size_coefficient,initial_lot_size,lot_multiplier,cooling_ratio,";
    headers += "profit_percent_to_close,incremental_profit_percent_per_position,rsi_period,rsi_buy_level,rsi_sell_level,";
    headers += "initial_pips_loss_quantity,pip_loss_quantity_increase,max_positions,close_all_at_max_positions,withdraw_at,";
    headers += "withdraw_amount,enable_limit_max_lot_size,dont_open_above_max_lot_size,max_lot_size,withdraw_total,";
    headers += "starting_lot_size,lot_size,positions_count,max_drawdown_percent,max_positions_opened,max_lot_size_opened,first_withdraw_at\n";
       
    // Apri il file in modalità FILE_WRITE e scrivi le intestazioni
    int write_handle = FileOpen(filename, FILE_WRITE | FILE_CSV | FILE_COMMON);
    if (write_handle != INVALID_HANDLE) {
        FileWriteString(write_handle, headers);
        FileClose(write_handle);
    } else {
        Print("Errore nell'apertura del file: ", GetLastError());
    }
    
}


........ other code not related ...............



double OnTester()
{
   
  end_date = TimeToString(TimeCurrent(), TIME_DATE);
  AppendToCSV();
 
   
 if (TesterStatistics(STAT_PROFIT)<0 || max_drawdown_percent>=100){
      return(0);
  } else {
     double  max_dd = max_drawdown_percent;
     
     return(withdraw_total);
  }
  
}



...................................


void AppendToCSV()
{
    // Definisci il nome del file CSV
    string filename = DoubleToString(current_version, 2) + "_" + _Symbol + ".csv";
    string path = custom_directory + filename;
    
    // Raccogli i dati
    string first_withdraw_date = TimeToString(first_withdraw_at, TIME_DATE);
    string date = TimeToString(GetTimeLocal(), TIME_DATE);
    string time = TimeToString(GetTimeLocal(), TIME_MINUTES);
    string timeframe_str = PeriodToString(PeriodSeconds()/60);

   // Raccogli i dati
    string data = StringFormat("%f,%s,%s,%s,%s,%f,%d,%d,%s,%s,", current_version, date, time, _Symbol, timeframe_str, initial_balance, AccountInfoDouble(ACCOUNT_BALANCE), AccountInfoInteger(ACCOUNT_LEVERAGE), begin_date, end_date);
    data += StringFormat("%d,%d,%f,%f,%f,", dynamic_initial_lot_size_enabled, dynamic_initial_lot_size_coefficient, initial_lot_size, lot_multiplier, cooling_ratio);
    data += StringFormat("%f,%f,%d,%f,%f,", profit_percent_to_close, incremental_profit_percent_per_position, rsi_period, rsi_buy_level, rsi_sell_level);
    data += StringFormat("%f,%f,%d,%d,%f,", initial_pips_loss_quantity, pip_loss_quantity_increase, max_positions, close_all_at_max_positions, withdraw_at);
    data += StringFormat("%f,%d,%d,%f,%f,", withdraw_amount, enable_limit_max_lot_size, dont_open_above_max_lot_size, max_lot_size, withdraw_total);
    data += StringFormat("%f,%f,%f,%f,", starting_lot_size, lot_size, positions_count, max_drawdown_percent);
    data += StringFormat("%d,%f,%s", max_positions_opened, max_lot_size_opened, first_withdraw_date);
    
    // Scrivi l'output su CSV
    int write_handle = FileOpen(filename, FILE_READ | FILE_WRITE | FILE_CSV | FILE_COMMON);
    if (write_handle != INVALID_HANDLE) {
              FileSeek(write_handle,0,SEEK_END);
         FileWriteString(write_handle, data + "\n");
         FileClose(write_handle);
    } else {
         Print("Errore nell'apertura del file: ", GetLastError());
    }
    
}