The historical data growth for the custom symbol is very rapid, possibly reaching several GBs in a single day.

New comment
 

I use the following code to import 1M real-time data, which means importing the data from the previous minute every minute. However, I find that the corresponding data on the disk grows extremely fast over the course of a day, and my disk space is simply not enough. Could you please let me know if there is any problem with my code?"


//+------------------------------------------------------------------+
//|                                                ImportHistory.mq5 |
//|                                  Copyright 2022, MetaQuotes Ltd. |
//|                                             https://www.mql5.com |
//+------------------------------------------------------------------+
#property copyright "Copyright 2022, MetaQuotes Ltd."
#property link      "https://www.mql5.com"
#property version   "1.00"


#include<ws/websocket.mqh>
#include<Defines.mqh>
#include<utils/LogUtils.mqh>

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
CWebsocket*       websocket = new CWebsocket();  //websocket client
int port = 3333;
string subscribeName = "klineImportMethod";
bool firstError = false;
//+------------------------------------------------------------------+
//| Script program start function                                    |
//+------------------------------------------------------------------+
void OnStart() {
//---
    wsConnect();
    subscribe();
    destroy();
}

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void subscribe() {
    string str;
    MqlRates rates[];
    MqlRates mqlRate;

    while(true) {
        if(websocket.ReadString(str)) {
//            string lines[];
//
//            if(StringLen(str) == 0 || StringSplit(str, '\n', lines) == -1 || lines.Size() != 2) {
//                logInfo("invalid request : " + str, LOG_LOCATION);
//                continue;
//            }
//
//            string kline = lines[1];
            string klines[];

            if(StringLen(str) == 0 || StringSplit(str, '\n', klines) == -1 || klines.Size() < 2) {
                logInfo("empty kline import request : " + str, LOG_LOCATION);
                continue;
            }

            int sizeOfKline = klines.Size() - 1;

            ArrayFree(rates);
            ArrayResize(rates, sizeOfKline);

            string symbol = "";
            datetime start_time;
            datetime end_time;

            for(int i=0; i<sizeOfKline; i++) {
                string kline = klines[i+1];
                string params[];

                if(StringLen(kline) == 0 || StringSplit(kline, '\t', params) == -1 || params.Size() != 7) {
                    logInfo("invalid kline : " + kline, LOG_LOCATION);
                    continue;
                }

                mqlRate.time = StringToTime(params[1]);
                mqlRate.open = StringToDouble(params[2]);
                mqlRate.high = StringToDouble(params[3]);
                mqlRate.low = StringToDouble(params[4]);
                mqlRate.close = StringToDouble(params[5]);
                mqlRate.tick_volume = 1;
                mqlRate.real_volume = 1;
                mqlRate.spread =1;

                rates[i] = mqlRate;

                if(i==0) {
                    start_time = mqlRate.time;
                    symbol = params[0];
                }
                if(i==sizeOfKline-1) {
                    end_time=mqlRate.time;
                }
            }

            if(!SymbolInfoInteger(symbol, SYMBOL_EXIST)) {
                if(!CustomSymbolCreate(symbol)) {
                    logInfo(StringFormat("error create custom symbol : %s", symbol), LOG_LOCATION);
                    return;
                }
            }
            if(!SymbolSelect(symbol, true)) {
                logInfo(StringFormat("error to add market watch : %s", symbol), LOG_LOCATION);
                return;
            }

            //ResetLastError();
            if(CustomRatesReplace(symbol, start_time, end_time, rates) == -1 && !firstError) {
                //firstError= true;
                //Print("error" + GetLastError() + " " + kline);
                //ResetLastError();
                //logInfo(StringFormat("error to update rates : %s, kline : %s", symbol, kline), LOG_LOCATION);
                logInfo(StringFormat("error to update rates : %s, %s-%s", symbol, TimeToString(start_time), TimeToString(end_time)), LOG_LOCATION);
            }
        }  else {
            wsConnect();
        }
    }
}

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void wsConnect() {

    destroy();
    websocket = new CWebsocket();

    while(!websocket.Connect("localhost", port, "name", false)) {
        printf("%s, %s:%d,", __FUNCTION__, websocket.LastErrorMessage(), websocket.LastError());
        Sleep(4000);
    }

    Print(__FUNCTION__, "socket connected!");

    if(!websocket.SendString(subscribeName)) {
        printf("%s, send error, %s:%d, ", __FUNCTION__, websocket.LastErrorMessage(), " : ", websocket.LastError());
        MessageBox("WebSocket Send Error!");
    }
}

//+------------------------------------------------------------------+
//|                                                                  |
//+------------------------------------------------------------------+
void destroy() {
    printf("%s", __FUNCTION__);

    if(websocket != NULL) {
        websocket.Close();
        delete websocket;
    }
}
//+------------------------------------------------------------------+
 
Your code appears to be replacing the existing rate data for the specified symbol and time range using the  CustomRatesReplace()  function. This means that every minute, the entire history of the previous minute is being stored, leading to a significant increase in disk space usage over time.
 
CustomRatesReplace : Fully replaces the price history of the custom symbol within the specified time interval with the data from the MqlRates type array.

It looks like you are using CustomRatesReplace where data from websocket is delivering more updates than 1 update in a 1 minute candle , If you are getting live data then there can be thousands of updates in 1 candle that is why its increasing the size of hst file. If you update data only 1 time in 1 minute  (only listens 1 frame of Kline in 1 minute) then you can significantly reduce the size of hst file.

 
Rong Zeng:

I use the following code to import 1M real-time data, which means importing the data from the previous minute every minute. However, I find that the corresponding data on the disk grows extremely fast over the course of a day, and my disk space is simply not enough. Could you please let me know if there is any problem with my code?"


Yes, your script is continuously receiving real-time data via WebSocket and storing it on the disk. This can indeed lead to rapid growth in the size of the data stored on disk, especially if the script is running continuously like this without any data management mechanisms in place, so you should check that

 
Oleksandr Medviediev #:
Your code appears to be replacing the existing rate data for the specified symbol and time range using the  CustomRatesReplace()  function. This means that every minute, the entire history of the previous minute is being stored, leading to a significant increase in disk space usage over time.
Thank you for your response. My update actually only updates one OHLC data point for the previous minute every minute, so it's actually only 1440 candlesticks per day, it shouldn't grow that fast. 
 
Arpit T #:
CustomRatesReplace : Fully replaces the price history of the custom symbol within the specified time interval with the data from the MqlRates type array.

It looks like you are using CustomRatesReplace where data from websocket is delivering more updates than 1 update in a 1 minute candle , If you are getting live data then there can be thousands of updates in 1 candle that is why its increasing the size of hst file. If you update data only 1 time in 1 minute  (only listens 1 frame of Kline in 1 minute) then you can significantly reduce the size of hst file.

Thank you for your response. My understanding of this interface is that it replaces the original data, meaning that the data corresponding to the same time point can be continuously replaced, but the data that should ultimately be stored in the database should be the last data. The data on the disk should not grow this fast. If my understanding is incorrect, could you please let me know what interface I should use to implement real-time import of candlestick data? In reality, I only update once per minute for a single symbol, 1440 times a day, and the space still grows quite fast. I really want to know how Terminal itself manages to update 1M data in real-time while taking up very little space. 
 
Nardus Van Staden #:

Yes, your script is continuously receiving real-time data via WebSocket and storing it on the disk. This can indeed lead to rapid growth in the size of the data stored on disk, especially if the script is running continuously like this without any data management mechanisms in place, so you should check that

Thank you for your response. Yes, I originally intended to implement a real-time candlestick update similar to Terminal's native trading pairs, but that would take up more space. Now, I'm updating only once per minute for a single symbol, but it still takes up a lot of space. If I want to implement the update method that Terminal has, how should I go about it? 
 
Rong Zeng #:
Thank you for your response. My understanding of this interface is that it replaces the original data, meaning that the data corresponding to the same time point can be continuously replaced, but the data that should ultimately be stored in the database should be the last data. The data on the disk should not grow this fast. If my understanding is incorrect, could you please let me know what interface I should use to implement real-time import of candlestick data? In reality, I only update once per minute for a single symbol, 1440 times a day, and the space still grows quite fast. I really want to know how Terminal itself manages to update 1M data in real-time while taking up very little space. 

Run this and check hcc file size

   MqlRates rates[];
   int nums = iBars(_Symbol, PERIOD_CURRENT);
   nums = nums>100000? 100000 : nums;
   if(nums == CopyRates(_Symbol, PERIOD_CURRENT, 0, nums, rates))
     {
      if(nums == CustomRatesDelete(_Symbol, rates[0].time, rates[nums-1].time))
        {
         CustomRatesUpdate(_Symbol, rates);
        }
     }
New comment