Web Scraping & data miningWeb Scraping & data miningweb scraping, data extraction, data mining and related things
As developers scrape data off the web, we use Node.js along with handy Cheerio scraper. When fetching .html() Cheerio parser returns the special symbols as HTML encoded entities, eg.:ä as ä ß as ß
(1) It’s not the job of a parser to preserve the original document.
(2) .html() returns an HTML representation of the parsed document, which doesn’t have to be equal to the original document.
source.
Imstall mariadb package:
npm i mariadb
const config = require("./config");
const db = config.database;
const mariadb = require('mariadb');
const pool = mariadb.createPool({
host: db.host,
user: db.user,
password: db.password,
database: db.database,
connectionLimit: 5
});
async function asyncSaveDataDB(data) {
let conn;
try {
conn = await pool.getConnection();
const rows = await conn.query("SELECT 1 as val");
console.log(rows); //[ {val: 1}, meta: ... ]
const res = await conn.query("INSERT INTO test (string1) value (?)", [data]);
console.log(res); // { affectedRows: 1, insertId: 1, warningStatus: 0 }
} catch (err) {
throw err;
} finally {
if (conn) return conn.end();
}
}
async function asyncSaveDataBulkDB(arr) {
let conn;
try {
conn = await pool.getConnection();
conn.batch("INSERT INTO `test` (string1) values (?)", arr)
.then(res => {
console.log(res); // 2
});
} catch (err) {
throw err;
} finally {
if (conn) return conn.end();
}
}
if (module.parent) {
module.exports = { asyncSaveDataDB, asyncSaveDataBulkDB }
} else {
asyncSaveDataBulkDB(['tt6', 'test 8']);
}Config.js might look like the following:
module.exports = {
database:{
host: "185.221.154.249",
user: "xxxxxxxxx",
password: "xxxxxxxxx",
database: 'xxxxxxxxx'
}
}I’ve mariadb installed on my VDS with Centos 7.
I’ve installed mysql npm package:
npm i mysqlYet requesting that package with
var mysql = require('mysql');
has not provided to the successful connection.
While
var mysql = require('mariadb');
has done.
var mysql = require('mariadb');
var con = mysql.createConnection({
host: "localhost",
user: "admin_default",
password: "xxxxxx",
database: 'admin_default'
}).then(function (){
console.log('connected!');
}, function(err){
console.log(err);
});
//console.log(con);
Given: a webpage to scrape.
If you inspect the DOM tree of that page you will find that quite a few tags are having the keyword dist. As an example:
<link rel="shortcut icon" type="image/x-icon" href="/wcsstore/ColesResponsiveStorefrontAssetStore/dist/30e70cfc76bf73d384beffa80ba6cbee/img/favicon.ico"><link rel="stylesheet" href="/wcsstore/ColesResponsiveStorefrontAssetStore/dist/30e70cfc76bf73d384beffa80ba6cbee/css/google/fonts-Source-Sans-Pro.css" type="text/css" media="screen">
See the scripts below.
function areCookiesEnabled()
{
var cookieEnabled = (navigator.cookieEnabled) ? true : false;
if (typeof navigator.cookieEnabled == "undefined" && !cookieEnabled)
{
document.cookie = "test";
cookieEnabled = (document.cookie.indexOf("test") != -1) ? true : false;
}
return cookieEnabled;
}Navigator is the interface represents the state and the identity of the user agent. It allows scripts to query it and to register themselves to carry on some activities.
A Navigator object can be retrieved using the read-only window.navigator property.
function isStorageEnabled() {
try{
sessionStorage.setItem("test","value");
if(sessionStorage.getItem("test") == "value") {
sessionStorage.removeItem("test");
return true;
} else {
return false;
}
} catch(err) {
return false;
}
}I’ve made a simple node.js server at VDS:
var http = require('http');
http.createServer(function (req, res) {
let port = 9999;
res.writeHead(200, {'Content-Type': 'text/plain'});
res.end('Hello World\n');
}).listen(post, '0.0.0.0');
console.log('Server running at port ' + port);
It works outputting:
Server running at port 9999
yet I can’t reach it at VPS/VDS IP where the code is residing: http://webscraping.pro:9999/ How to solve that?
We recently composed a scraper that works to extract data of a static site. By a static site, we mean such a site that does not utilize JS scripting that loads or transforms on-site data.
When working with Apify crawlers, it’s necessary to init RequestQueue. How to fill in RequestQueue from txt file?
A text file with urls to crawl. In our case it’s categories.txt. We’ll use LineReader node package to open and iterate the file line by line.
LineReader to install:
npm i --save line-reader Since requestQueue methods return Promise, when iterating over the lines of the file we need to apply async function for each line to be added as url into the requestQueue.
const queue_name ='ebinger';
const base_url = 'https://www.ebinger.com/';
Apify.main(async () => {
const requestQueue = await Apify.openRequestQueue(queue_name);
const lineReader = require('line-reader');
lineReader.eachLine('categories.txt', async function(line) {
//console.log('adding ', line);
let url = base_url + line.trim();
await requestQueue.addRequest({ url: url });
});
var { totalRequestCount, handledRequestCount, pendingRequestCount, name } = await requestQueue.getInfo();
console.log(`RequestQueue "${name}" with requests:` );
console.log(' handledRequestCount:', handledRequestCount);
console.log(' pendingRequestCount:', pendingRequestCount);
console.log(' totalRequestCount:' , totalRequestCount);
...