In C, using gumbo and libcurl you could do something like this to clone the website. Pardon the code as it is quite sloppy and will need modified for PHP and other resources.
Code:
#include <iostream>
#include <cstring>
#include <curl/curl.h>
#include <vector>
#include <sys/types.h>
#include <sys/stat.h>
#include "gumbo.h"
using namespace std;
CURL *curl;
CURLcode res;
int i = 1;
string data;
vector<string> links;
static size_t callback(void *data, size_t size, size_t nmemb, void *pointer) {
( (string*)pointer)->append((char*)data, size * nmemb);
return size * nmemb;
}
string ltrim(string s, const char* t = " \t\n\r\f\v"){
s.erase(0, s.find_first_not_of(t));
return s;
}
string rtrim(string s, const char* t = " \t\n\r\f\v"){
s.erase(s.find_last_not_of(t) + 1);
return s;
}
string trim(string s, const char* t = " \t\n\r\f\v"){
return ltrim(rtrim(s, t), t);
}
void extract_links( GumboNode* node )
{
GumboAttribute* detail;
if (node->type != GUMBO_NODE_ELEMENT) {
return;
}
if( node->v.element.tag == GUMBO_TAG_A &&
(detail = gumbo_get_attribute(&node->v.element.attributes, "href"))){
links.push_back( trim( detail->value ) );
}
GumboVector* children = &node->v.element.children;
for (unsigned int i = 0; i < children->length; ++i) {
extract_links(static_cast<GumboNode*>(children->data[i]));
}
}
int initialize_curl( const char* url )
{
curl_global_init(CURL_GLOBAL_ALL);
curl = curl_easy_init();
if(curl) {
curl_easy_setopt(curl, CURLOPT_URL, url);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, callback);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &data);
curl_easy_setopt(curl, CURLOPT_USERAGENT, "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36");
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt(curl, CURLOPT_COOKIEFILE, "cookies.txt");
curl_easy_setopt(curl, CURLOPT_COOKIEJAR, "cookies.txt");
res = curl_easy_perform(curl);
if(res != CURLE_OK) {
curl_easy_strerror(res);
return 1;
}
curl_easy_cleanup(curl);
}
return 0;
}
void write_to_html_out( const char* input, const char* filename )
{
string core = "/home/user/Downloads";
char buffer[260];
FILE *pFile;
if(strstr( filename, "http") == NULL ) {
if( strstr( filename, "html") == NULL ) {
string tempfile = filename;
string outfile = tempfile.substr(0, tempfile.find_last_of("/") );
outfile = core + outfile;
mkdir(outfile.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
}
}
sprintf( buffer, "%s%s", core.c_str(), filename );
if ( ( pFile = fopen(buffer, "w+") ) != NULL) {
fprintf(pFile, "%s", input );
fclose(pFile);
}
}
string replace_all(string str, const string& from, const string& to) {
size_t start_pos = 0;
while((start_pos = str.find(from, start_pos)) != std::string::npos) {
str.replace(start_pos, from.length(), to);
start_pos += to.length();
}
return str;
}
void cleanup() { data = ""; }
int main (int argc, char *argv[])
{
string root_url = "http://www.thedraculasociety.org.uk";
if( initialize_curl( root_url.c_str() ) == 0 )
{
GumboOutput* output = gumbo_parse(data.c_str());
extract_links(output->root);
write_to_html_out( data.c_str(), "/index.html" );
for(vector<int>::size_type i = 0; i != links.size(); i++)
{
string url = root_url + links[i].c_str();
cleanup( );
printf("Grabbing link: %s\r\n", links[i].c_str() );
if( initialize_curl( url.c_str() ) == 0 ) {
GumboOutput* output = gumbo_parse(data.c_str());
write_to_html_out( data.c_str(), links[i].c_str() );
cleanup( );
}
}
getchar( );
}
return 0;
}
![[Image: ZFxRuwc.png]](http://i.imgur.com/ZFxRuwc.png)
![[+]](https://sinister.ly/images/modern/collapse_collapsed.png)
