Can't scrape vix from can

[[Deleted]]

Hello Guys,

hope u can help...

trying to scrape vix level from cnn.

Unfortunately this is the result:

I do not understand why nothing is found.

Thank you for your help!

This is my code:

//+------------------------------------------------------------------+

//|                                          Fear & Greed Webscraper |

//|                                         Copyright 2022, |

//|                                       |

//+------------------------------------------------------------------+

//------------------------------importing wininet.dll and prototyping the functions that will be called

#import "wininet.dll"

int InternetCheckConnectionW(string &lpszUrl,uint dwFlags,uint dwReserved);

int InternetOpenW(string &lpszAgent,uint dwAccessType,string &lpszProxyName,string &lpszProxyBypass,uint dwFlags);

int InternetOpenUrlW(int hInternetSession,string &lpszUrl,string &lpszHeaders,uint dwHeadersLength,uint dwFlags,uint dwContext);

int InternetConnectW(int hInternet,string &lpszServerName,int nServerPort,string &lpszUsername,string &lpszPassword,uint dwService,uint dwFlags,uint dwContext);

int HttpOpenRequestW(int hConnect,string &lpszVerb,string &lpszObjectName,string &lpszVersion,string &lpszReferer,string &lplpszAcceptTypes,uint dwFlags,uint dwContext);

int HttpSendRequestW(int hRequest,string &lpszHeaders,uint dwHeadersLength,uchar &lpOptional[],uint dwOptionalLength);

int InternetReadFile(int hFile,uchar &lpBuffer[],uint dwNumberOfBytesToRead,uint &lpdwNumberOfBytesRead);

int InternetCloseHandle(int hInternet);

#import

uchar uc_Buffer[1640000000], uc_DynBuf[]; // InternetReadFile() expects a static buffer.  The size of uc_Buffer is set to accommodate the specific number of bytes that will be downloaded.. will receive the html text downloaded from the web server

float VIX;

//------------------------------------------------------------------------------------------------------

//------------------------------------------first we check to see if an internet connection is available.

void OnInit()

  {

   bool bResult;  int i,iNet1,iNet2; //, iRequest;  

   

   string stURL="http://www.msn.com";

   bResult=InternetCheckConnectionW(stURL,1,0); // 1 == FLAG_ICC_FORCE_CONNECTION

   Print("InternetCheckConnectionW() returned ",bResult);

   if(!bResult) return;

   string stAgent="Mozilla/5.0",stNull="";

   iNet1 = InternetOpenW(stAgent, // _In_ LPCTSTR lpszAgent 

                         1,       // 1 == INTERNET_OPEN_TYPE_DIRECT

                         stNull,  // _In_ LPCTSTR lpszProxyName

                         stNull,  // _In_ LPCTSTR lpszProxyBypass

                         NULL);   // _In_ DWORD dwFlags

   Print("iNet1 == ",iNet1);

   if(iNet1==0) return;

//----------------------------------------------------------------------------------------------

//------------------------------------------a connection to the web server is established, initializing the handle iNet2 to download the html file.

   stURL="https://edition.cnn.com/markets/fear-and-greed";

   string stHdr="Accept: text/*";

   iNet2 = InternetOpenUrlW(iNet1,            // HINTERNET hInternet,

                            stURL,            // LPCWSTR   lpszUrl,

                            stHdr,            // LPCWSTR   lpszHeaders,

                            StringLen(stHdr), // DWORD     dwHeadersLength,

                            0x00080000,       // DWORD dwFlags, 0x00080000 == INTERNET_FLAG_NO_COOKIES

                            NULL);            // DWORD_PTR dwContext

   Print("iNet2 == ",iNet2);

   if(iNet2==0)

     {

      InternetCloseHandle(iNet1);

      return;

     }

//----------------------------------------------------------------------------------------------

//-------------------------------------------------Download data from the web server.

  uint uGet, uGot, uDst, uMax;

  uGet = 1638400000;    // number of bytes to download per call to InternetReadFile, must be at least 1 byte less than size of uc_Buffer

  uGot = uDst = 0; // uGot is number of bytes downloaded in call to InternetReadFile; uDst is total number of bytes downloaded.

  uMax = 18022400000;   // maximum number of bytes to download.

  do

  { bResult = InternetReadFile(iNet2,     // _In_  HINTERNET hFile

                               uc_Buffer, // _Out_ LPVOID lpBuffer

                               uGet,      // _In_  DWORD dwNumberOfBytesToRead

                               uGot);     // _Out_ LPDWORD lpdwNumberOfBytesRead

    uc_Buffer[uGot] = 0; // Terminate string in uc_Buffer by appending a null character.

    ArrayCopy(uc_DynBuf, // destination array 

              uc_Buffer, // source array 

              uDst,      // index at which writing to destination array begins 

              0,         // index at which copying from source array begins 

              uGot);     // number of elements to copy 

    uDst += uGot; // advance index of destination array for next pass in loop

  Print( " uc_Buffer:", CharArrayToString(uc_Buffer)); 

  }while(bResult && uGot > 0 && uDst < uMax);

 

  Print( "Size of uc_DynBuf == ", ArraySize(uc_DynBuf));

  Print("Bytes downloaded  == ", uDst);   

 //----------------------------------------------------------------------------------------------

 

 

 

 

 

 

 //-----------------------------Now we search for the meta tag of interest in the downloaded text

  i = StringFind(CharArrayToString(uc_DynBuf), "<span class=\"market-fng-gauge__dial-number-value\">", 0); // 0 == position from which search starts 

  Print("<span class=\"market-fng-gauge__dial-number-value\"> == ", i); 

  if(i == -1) {Print("<span class=\"market-fng-gauge__dial-number-value\"> not found.");  InternetCloseHandle(iNet1);  return;}

   i = i + 52; // Advance index to known location of text representing vix.

   Print("i= ", i);

  VIX = StringToDouble(StringSubstr(CharArrayToString(uc_Buffer), i, 2));

  Print("foundstring: ", StringSubstr(CharArrayToString(uc_Buffer), i, 2));

  Print("VIX: ", VIX);

  InternetCloseHandle(iNet1); // Done with wininet.

}//END void OnStart()

//----------------------------------------------------------------------------------------------

void OnTick()

{

    Comment("VIX:",VIX);

}

[Carl Schreiber]

Please edit your posted code with the code bottom: Alt+s or the </> button from the header line.

To dld. from the internet you don't need the Windows functions (with example): https://www.mql5.com/de/docs/network/webrequest

If the code compiles without any error but doesn't do what it should you can use the debugger - a very powerful tool:

https://www.metatrader5.com/en/metaeditor/help/development/debug // Code debugging
https://www.mql5.com/en/articles/2041 // Error Handling and Logging in MQL5
https://www.mql5.com/en/articles/272 // Tracing, Debugging and Structural Analysis of Source Code
https://www.mql5.com/en/articles/35 // scroll down to: "Launching and Debugging"

(btw if you replace /en/ by /de/ in the MQ-Urls you'll get the German version).

[Jan4654]

Scraping is easily detected with webrequests especially on big commercial sites like cnn and like.

Simple requests can not excute js code and or ip or asn belong to datacenter == detected your crap web crawler.

They will let web crawlers like google search engine and other friendly web crawlers poke around more freely.