NLP: Natural Language Processing for posts on STEEMIT (experimental, nodejs)

in Steem Marketing2 years ago

Natural Language Processing.png

It is very interesting. Natural Language Processing or NLP makes it possible to automatically process and analyze texts. There are ready-made powerful tools for this.

For example:
https://openbase.com/js/sentiment

Returned Objects
Score: Score calculated by adding the sentiment values of recognized words.
Comparative: Comparative score of the input string.
Calculation: An array of words that have a negative or positive valence with their respective AFINN score.
Token: All the tokens like words or emojis found in the input string.
Words: List of words from input string that were found in AFINN list.
Positive: List of positive words in input string that were found in AFINN list.
Negative: List of negative words in input string that were found in AFINN list.

Script text20.js


Script that parses all posts by tag and
adds them to the table, based on mood.

const mysql = require("mysql2");
const steem = require('steem');
const fs = require("fs");            
  
var settings = require('./config.js');

let body3R = '';
let num_day = settings.num_day;
let bd_name = settings.bd_name;
let hive_name = settings.hive_name;
let url_post = settings.url_post;
let password_mysql = settings.password_mysql;
let user_mysql = settings.user_mysql;
let title_name = settings.title_name;
let image_list = settings.image_list;
let wifkey = settings.wifkey;
let votey = settings.account;

config = {
  host: "localhost",
  user: user_mysql,
  database: bd_name,
  password: password_mysql
}

const connection = mysql.createConnection(config);

    var d = new Date(); // Today!
    d.setDate(d.getDate()-num_day); // Yesterday!
    d = d.toJSON().split("T")[0];
    d = d.replace(/-/gi, '');
    console.log(d);   


let sql_sentiment = 'create table if not exists `analyze`(id int primary key auto_increment,title varchar(255), author varchar(255), url varchar(255) UNIQUE KEY, score varchar(255), comparative varchar(255), positivewords varchar(255), negativewords varchar(255))';

connection.query(sql_sentiment, function(err, results) {
if(err) console.log(err);
else console.log("Table sentiment создана");
});

 // index.js
const dsteem = require('dsteem');
const client = new dsteem.Client('https://api.steemit.com');
var natural = require('natural');
var tokenizer = new natural.WordTokenizer();
 
 function sortByCount(array)
{
    if( !Array.isArray(array) )
        throw new TypeError('Incorrect type of ' + typeof array);

    let valuesMap = new Map();
    array.forEach(elem => {

        valuesMap.set(elem, valuesMap.has(elem) ? valuesMap.get(elem) + 1 :  1);
    });

    let arr =  [...valuesMap.entries()].sort((a, b) => b[1] - a[1]);

    return arr.map(value => value[0]);
}


    const filter = "created";
    const query = {tag: "",limit: 1}
 
    console.log('Post assembled.\nFilter:', filter, '\nQuery:', query);

    client.database
        .getDiscussions(filter, query)
        .then(result => {
            console.log('Response received:', result);
            if (result) {
                var posts = [];
                result.forEach(post => {
                    const json = JSON.parse(post.json_metadata);
                    const image = json.image ? json.image[0] : '';
                    const title = post.title;
                    const author = post.author;
                    const created = new Date(post.created).toDateString();
                    let upvote = post.active_votes.length;

            
//console.log(post.body);
let text2 = tokenizer.tokenize(post.body);
//console.log(text2);

// исходный массив
let fruits = ['banana', 'fruit', 'apple', 'apple', 'orange', 'banana', 'orange', 'banana', 'banana', 'apple'];

var Sentiment = require('sentiment');
var sentiment = new Sentiment();
var result = sentiment.analyze(post.body);
console.log('******');  
console.log('******'); 
console.dir(result);   
console.log('******');  
console.log('******'); 

let sql_sentiment = 'create table if not exists `analyze`(id int primary key auto_increment,title varchar(255), author varchar(255), url varchar(255) UNIQUE KEY, score varchar(255), comparative varchar(255), positivewords varchar(255), negativewords varchar(255))';
            
    const sql_insert = "INSERT INTO `analyze`(`title`, `author`, `url`, `score`, `comparative`, `positivewords`, `negativewords`) VALUES('"+title+"', '"+author+"', '"+post.url+"','"+result.score+"', '"+result.comparative+"', '"+result.positive+"', '"+result.negative+"')";    
    
    //const sql_insert = "INSERT INTO `analyze`(`title`) VALUES('${title}')";       


                    
        connection.query(sql_insert, function(err, results) {
            if(err) console.log(err);
        console.log(results);
                }); 
                
let ex = sortByCount (text2);
//console.log(ex);

                });

            } else {
                console.log('No result');

            }
        })
        .catch(err => {
            console.log(err);
        });


Now I need to make a script to display a list from the table and form a post.

Script list_sentimental.js

const mysql = require("mysql2");
  

var settings = require('./config.js');

let body3R = '';
let num_day = settings.num_day;
let bd_name = settings.bd_name;
let hive_name = settings.hive_name;
let url_post = settings.url_post;
let password_mysql = settings.password_mysql;
let user_mysql = settings.user_mysql;
let title_name = settings.title_name;
let image_list = settings.image_list;

config = {
  host: "localhost",
  user: user_mysql,
  database: bd_name,
  password: password_mysql
}

const connection = mysql.createConnection(config);


let formula;

    var d = new Date(); // Today!
    d.setDate(d.getDate()-num_day); // Yesterday!
    d = d.toJSON().split("T")[0];
    d = d.replace(/-/gi, '');
    
let mytable = 'analyze';
let name_file = mytable+d+".txt";


console.log(mytable);

let full = `
pic


This is an experimental sentiment analysis of STEEMIT posts. You can read the development details in the post at the link:

<h2>Experimental Post Mood Analysis</h2>
|title|score|c.|positive words|negative words|
--------|-----|-----|-----|-----|`;

const sql = 'SELECT * FROM `'+mytable+'` ORDER BY SCORE';

    console.log(sql);

connection.query(sql,  function(err, results) {
    if(err) console.log(err);
    const users = results;
    console.log('users');
    console.log('users');
    console.log('users');
    console.log('users');
    console.log('users');
    console.log(users);
    console.log('users');
    console.log('users');
    console.log('users');
    console.log('users');
    console.log('users');

    const fs = require("fs");   
    
fs.appendFileSync(name_file,full);

     for(let i=0; i < users.length; i++){   
        {
                     
            let notfull;
            var d = new Date(); // Today!
            d.setDate(d.getDate()-num_day); // Yesterday!
            d = d.toJSON().split("T")[0];
                d = d.replace(/-/gi, '');
 
 title = users[i].title.replace(/[^a-zа-яё0-9\s]/gi, ' ');
 
  let positivewords = users[i].positivewords;
  positivewords = positivewords.replace(',', ' ');
  positivewords = positivewords.replace(',', ' ');
  positivewords = positivewords.replace(',', ' ');
  positivewords = positivewords.replace(',', ' ');
  positivewords = positivewords.replace(',', ' ');
  positivewords = positivewords.replace(',', ' ');
  let negativewords = users[i].negativewords;
  negativewords = negativewords.replace(',', ' ');
  negativewords = negativewords.replace(',', ' ');
  negativewords = negativewords.replace(',', ' ');
  negativewords = negativewords.replace(',', ' ');
  negativewords = negativewords.replace(',', ' ');
  
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);
    console.log(positivewords);

            console.log('|',i+1,'|',title,'|',users[i].author,'|',users[i].url,'|',users[i].score,'|',users[i].comparative,'|',positivewords,'|',negativewords,'|');

            let z = Number(i + 1);
            
let comparative = users[i].comparative;
 comparative = 0;
//parseFloat(comparative.toFixed(3));

notfull = '\n|<a href="'+users[i].url+'">'+title+'</a>|'+users[i].score+'|'+comparative+'|'+positivewords+'|'+negativewords+'|';
            
           
        fs.appendFileSync(name_file,notfull);


         }

     }

            let notfull;
            var d = new Date(); // Today!
            d.setDate(d.getDate()-num_day); // Yesterday!
            d = d.toJSON().split("T")[0];
                d = d.replace(/-/gi, '');


full = `


Thanks for attention!
`


      fs.appendFileSync(name_file,full);
        
    
});
 
connection.end();

Have a nice day!

Sort:  

Upvoted! Thank you for supporting witness @jswit.

Coin Marketplace

STEEM 0.35
TRX 0.12
JST 0.039
BTC 69796.92
ETH 3521.66
USDT 1.00
SBD 4.70