One Hat Cyber Team

View File Name : grab.js

//create new webpage object

// Return array of string values, or NULL if CSV string not well formed.
 function CSVToArray( strData, strDelimiter ){
	strDelimiter = (strDelimiter || ",");
	var objPattern = new RegExp(
		(
			"(\\" + strDelimiter + "|\\r?\\n|\\r|^)" +
			"(?:\"([^\"]*(?:\"\"[^\"]*)*)\"|" +
			"([^\"\\" + strDelimiter + "\\r\\n]*))"
		),
		"gi"
		);
	var arrData = [[]];
	var arrMatches = null;
	while (arrMatches = objPattern.exec( strData )){
		var strMatchedDelimiter = arrMatches[ 1 ];
		if (
			strMatchedDelimiter.length &&
			(strMatchedDelimiter != strDelimiter)
			){
			arrData.push( [] );
		}
		if (arrMatches[ 2 ]){
			var strMatchedValue = arrMatches[ 2 ].replace(
				new RegExp( "\"\"", "g" ),
				"\""
				);
		} else {
			var strMatchedValue = arrMatches[ 3 ];
		}
		arrData[ arrData.length - 1 ].push( strMatchedValue );
	}
	return( arrData );
}

var fs = require('fs');
var csvf = fs.open(fs.workingDirectory + '/top-1m.csv/top-1m.csv_Pieces/top-1m_2.csv', 'r');
var csvready = fs.open(fs.workingDirectory + '/top-1m.csv/ready_2.csv', 'r');

var readylist = [];
var worklist;

while(!csvready.atEnd()){
	readylist.push(CSVToArray(csvready.readLine(), ';'));
}
while(!csvf.atEnd()){
	var tmpar = csvf.readLine().split(',');
	console.log(tmpar[0]);
	worklist.parse({ 'url' : tmpar[1], 'id' : tmpar[0] });
}

console.log(worklist);

var RenderUrlsToFile, arrayOfUrls, system;

system = require("system");

RenderUrlsToFile = function(sites, callbackPerUrl, callbackFinal) {
    var getFilename, next, page, retrieve, webpage;
    webpage = require("webpage");
    page = null;
    getFilename = function(url) {
        return "newimages/" + url + ".jpg";
    };
    next = function(status, url, file) {
        page.close();
        callbackPerUrl(status, url, file);
        return retrieve();
    };
    retrieve = function() {
        var site;
        if (sites.length > 0) {
            site = sites.shift();
            
            var url = site['url'];
            page = webpage.create();
            page.viewportSize = {width: 1024, height: 500};
            page.clipRect = { top: 0, left: 0, width: 1024, height: 500 };
            page.settings.userAgent = "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.94 Safari/537.36";
            page.customHeaders = {
			    'Accept-Language': 'en-US;q=0.6,en;q=0.4'
			};
			page.onConsoleMessage = function (msg) { console.log(msg); };
            return page.open("http://" + url, function(status) {
                var file;
                file = getFilename(url);
                if (status === "success") {
                    page.render(file);
                    var pagedata = page.evaluate(function () {
                    	var ptitle = document.querySelector("title").innerText;
                    	var descr = document.querySelector("meta[name=description]");
                    	var pdescr = '', pkeywd = '';
                    	if(descr !== null){
                    		 pdescr = descr.getAttribute("content");
                    	}
                    	var keywd = document.querySelector("meta[name=keywords]");	
                    	if(keywd !== null){
                    		 pkeywd = keywd.getAttribute("content");
                    	}
					    return {"title" : ptitle, "description" : pdescr, 'keywords' : pkeywd};
					});
					var savesite = {
						'id' : site['id'],
						'title' : pagedata.title,
						"description" : pagedata.description,
						"keywords" : pagedata.keywords,
						"prefix" : 'http://',
						"redirect" : url,
						"alexa_rank" : site['id'],
						"mainimg" : url + ".jpg"

					}
					console.log(pagedata);
                    return next(status, url, file);
                } else {
                    return next(status, url, file);
                }
            });
        } else {
            return callbackFinal();
        }
    };
    return retrieve();
};

RenderUrlsToFile(worklist, (function(status, url, file) {
    if (status !== "success") {
        return console.log("Unable to render '" + url + "'");
    } else {
        return console.log("Rendered '" + url + "' at '" + file + "'");
    }
}), function() {
    return phantom.exit();
});