Development of automation for checking uniqueness in STEEMIT. Part 3. Run script posts & uniq
Итак, продолжаю. Задача, чтобы удалось запустить два скрипта - posts и uniq.
Первая задача. Необходимо добится запуска сканирования и загрузки постов в таблицу базы данных.
Захожу на https://dashboard.heroku.com/apps/gardenweb0301
Смотрю что на данный момент происходит в базе.
heroku login
heroku logs --tail
https://gardenweb0301.herokuapp.com/addpost
git add . && git commit -m "update pkg-lock" && git push heroku master
Вот что нашел тут: https://golos.id/ru--obrazovanie/@rusldv/zakanchivaem-kurs-php-i-delaem-web-klient-dlya-golosa
steem.api.getAccounts(['<?php echo $nick; ?>'], function(err, result) {
// balance
if(result.length <= 0) {
window.location.replace('http://golos.site/404.html');
}
$(".reputation").text(steem.formatter.reputation(result[0].reputation));
// profile
//console.log(err, result[0].json_metadata);
var json_meta = JSON.parse(result[0].json_metadata);
//console.log(err, json_meta.profile);
if('profile' in json_meta)
{
if(json_meta.profile.profile_image === undefined || json_meta.profile.profile_image === null || json_meta.profile.profile_image === "") {
$(".profile_image").attr("src", '/img/default.jpg');
} else {
$(".profile_image").attr("src", json_meta.profile.profile_image);
}
$(".profile_name").text(json_meta.profile.name);
$("title").text(json_meta.profile.name);
$(".profile_location").html('<i class="fa fa-map-marker"></i> ' + json_meta.profile.location);
$(".profile_about").text(json_meta.profile.about);
} else {
//console.log('profile is empty');
$(".profile_image").attr("src", '/img/default.jpg');
$(".profile_name").text('нету');
$(".profile_location").text('нету');
$(".profile_about").text('отсутствует');
}
// any
//console.log(result);
$(".post_count").text(result[0].post_count);
steem.api.getFollowCount('<?php echo $nick; ?>', function(err, result) {
//console.log(err, result);
$(".follower_count").text(result.follower_count);
$(".following_count").text(result.following_count);
});
//console.log(result[0].last_post);
steem.api.getDiscussionsByAuthorBeforeDate('<?php echo $nick; ?>', '', result[0].last_post, 10, function(err, result) {
//console.log(err, result);
result.forEach(addNewsItem);
});
});
function addNewsItem(item, i, arr) {
console.log(item);
var prevstr = item.body.substring(0, 392).replace(/<\/?[^>]+>/i,'');
//var re = /(https?:\/\/\[a-zA-Z0-9а-яА-Я\/\.-_]+\.jpg)/;
var re = /(https?:\/\/[a-zA-Zа-яА-Я0-9\.\/\-_=&%\?]+[jpg|png|gif])/g;
prevstr = prevstr.replace(re, '<-img class="img-responsive img-rounded" src=$1 />');
last_permlink = item.permlink;
$("#news").append('<div class="well"><h2>' + item.title + '</h2><p>' + prevstr + '...</p><p><small><a href="javascript:showItem(\'' + item.permlink + '\');"><i class="fa fa-arrow-circle-o-down"></i> Открыть</a> | <a href="https://golos.io/@<?php echo $nick; ?>/' + last_permlink + '" target="_blank"><i class="fa fa-rocket"></i> Поросмотреть на golos.io</a></small></p></div>');
//$(document).on('scroll', nextCommentScroll);
}
function showItem(data) {
alert(data);
}
В целом - и сейчас скрипт работает, хоть и со существеннми перебоями. Скрипт по добавлению постов в таблицу. За 3 запуска он добавил 14 постов - как и в моем случае на ноутбуке.
Теперь необходимо:
Добавить планировщик
Понять восстанавливается ли сервер сам
Проанализировать работу uniq-скрипта
Проанализировать работу spisok
Наладить работу spisok на своем ноуте
понять как работать с num.js
(...)
Итак, условно работает добавление постов в базу.
Теперь нужно наладить саму проверку уникальности.
Теперь uniq (unic) скрипт также настроен.
Ищу онлайн планировщки (online-cron)
Spisok и number также есть.
//create table if not exists heroku_cfdf255bd781813.post_number( id int primary key auto_increment, number int)
//UPDATE heroku_cfdf255bd781813.post_number SET number='0' LIMIT 1000
https://www.easycron.com/user - платный
Так, это все работает. Теперь вопрос восстанавливается ли само приложение.
И, конечно, надо будет менять скрипт с добавлением постов в будущем.
https://answer-id.com/ru/53599723
Тут описано как сделать автоматический перезапуск (на всякий случай, сейчас это мне важно).
https://console.cron-job.org/jobs/create - бесплатный
Итак, на данный момент все настроено и начинается этап тестирования.
Если удастся, то сейчас также подключу автоматическую публикацию отчета.
Все настроено, но из-за некоректности работы скриптов приложение (сервер) рушится (падает).
https://elements.heroku.com/addons/scheduler
Это я продолжу улучшать.
Однако сейчас уже можно сказать о первой работоспособной версии.
App:
var createError = require('http-errors');
var express = require('express');
var path = require('path');
var cookieParser = require('cookie-parser');
var logger = require('morgan');
var indexRouter = require('./routes/index');
var setRouter = require('./routes/set');
var usersRouter = require('./routes/users');
var createRouter = require('./routes/create');
var addpostRouter = require('./routes/addpost');
var spisokRouter = require('./routes/spisok');
var unicRouter = require('./routes/unic');
var numRouter = require('./routes/num');
var listRouter = require('./routes/list');
var postRouter = require('./routes/post');
var app = express();
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'pug');
app.use(logger('dev'));
app.use(express.json());
app.use(express.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));
app.use('/', indexRouter);
app.use('/users', usersRouter);
app.use('/set', setRouter);
app.use('/create', createRouter);
app.use('/addpost', addpostRouter);
app.use('/spisok', spisokRouter);
app.use('/unic', unicRouter);
app.use('/num', numRouter);
app.use('/list', listRouter);
app.use('/post', postRouter);
// catch 404 and forward to error handler
app.use(function(req, res, next) {
next(createError(404));
});
// error handler
app.use(function(err, req, res, next) {
// set locals, only providing error in development
res.locals.message = err.message;
res.locals.error = req.app.get('env') === 'development' ? err : {};
// render the error page
res.status(err.status || 500);
res.render('error');
});
app.all('/secret', function (req, res, next) {
console.log('Accessing the secret section ...');
next(); // pass control to the next handler
});
app.get('/about', function (req, res) {
res.send('about');
});
module.exports = app;
Routers:
addpost
var express = require('express');
var express = require('express');
var router = express.Router();
const fs = require("fs");
const steem = require('steem');
let body3R = '';
let num_day = 1;
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
d = d.replace(/-/gi, '');
console.log(d);
//let mytable = 'poststest';
let mytable = 'p'+d;
let mytable_p = mytable;
const mysql = require("mysql2");
const sql = `SELECT * FROM spisok`;
//const sql = `SELECT * FROM p0111 ORDER BY length DESC`;
// console.log(sql);
/* GET users listing. */
router.get('/', function(req, res, next) {
const connection = mysql.createConnection({
host: "us-cdbr-east-05.cleardb.net",
user: "b9ce03f75c865a",
database: "heroku_cfdf255bd781813",
password: "503a3e51"
});
connection.query(sql, function(err, results) {
if(err) console.log(err);
console.log(results[2]['author']);
users = results;
accounts = users;
let sql = `create table if not exists ${mytable_p}( id int primary key auto_increment, author varchar(255) not null, title varchar(255) not null, created varchar(255) not null, length int not null, url varchar(255) UNIQUE KEY, comments int not null, upvotes int not null, points float not null, percent float not null)`;
connection.query(sql, function(err, results) {
if(err) console.log(err);
console.log(results);
});
//https://steemit.com/hive-171319/@alexmove/the-word-game-post-2
//accounts.forEach(function (element,i,array)
len = accounts.length;
for (yyy = 0; yyy < len; yyy++)
{
console.log('yyy');
console.log(yyy);
//steem.api.getDiscussionsByAuthorBeforeDate(accounts[yyy]['author'],null, new Date().toISOString().split('.')[0],3, function(connection, err, result) {
steem.api.getDiscussionsByAuthorBeforeDate(accounts[yyy]['author'],null, new Date().toISOString().split('.')[0],3, function(err, result) {
console.log('yyy');
console.log(yyy);
if (result)
{
var i, len = result.length;
for (i = 0; i < len; i++)
{
var raw = result[i];
console.log(raw.author);
console.log(raw.created);
let upvote = raw.active_votes.length-1;
let input = raw.created;
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
let title = raw.title;
title = title.replace(/[^a-zа-яё0-9\s]/gi, ' ');
if ((input.indexOf(d) !== -1)&& (raw.category === 'hive-180821'))
{
const sql = `INSERT INTO ${mytable}(author, title, created, length, url, comments, upvotes, points,percent) VALUES('${raw.author}', '${title}', '${raw.created}', '${raw.body_length}', '${raw.url}', '${raw.children}', '${upvote}','0','0')`;
connection.query(sql, function(err, results) {
if(err) console.log(err);
console.log(results);
});
}
}
//connection.end();
}
});
}
});
});
module.exports = router;
create
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
const mysql = require("mysql2");
const connection = mysql.createConnection({
host: "us-cdbr-east-05.cleardb.net",
user: "b9ce03f75c865a",
database: "heroku_cfdf255bd781813",
password: "503a3e51"
});
let num_day = 1;
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
d = d.replace(/-/gi, '');
//res.send(d);
let mytable_p = 'posts01';
let mytable_c = 'c1711';
let mytable_day = 'a1711';
let sql = `create table if not exists spisok( id int primary key auto_increment, author varchar(255) UNIQUE KEY)`;
connection.query(sql, function(err, results) {
if(err) res.send(err);
//else //res.send("Таблица создана");
});
//create table if not exists heroku_cfdf255bd781813.post_number( id int primary key auto_increment, number int)
//UPDATE heroku_cfdf255bd781813.post_number SET number='0' LIMIT 1000
connection.end();
// res.send('create');
});
module.exports = router;
list
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
const mysql = require("mysql2");
let num_day = 1;
const connection = mysql.createConnection({
host: "us-cdbr-east-05.cleardb.net",
user: "b9ce03f75c865a",
database: "heroku_cfdf255bd781813",
password: "503a3e51"
});
let formula;
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
d = d.replace(/-/gi, '');
let mytable = 'p'+d;
let full = `
|N|acc|title|comm|length|uniq|
------------|-------------|------------|-------------|----|-----|`;
const fs = require("fs");
fs.appendFileSync("..\\report"+d+".txt",full);
const sql = `SELECT * FROM ${mytable} ORDER BY length DESC` ;
connection.query(sql, function(err, results) {
if(err) console.log(err);
const users = results;
for(let i=0; i < users.length; i++){
let notfull;
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
d = d.replace(/-/gi, '');
if (users[i].count>0) {
formula = (users.length - i)/10;
// console.log(users[i].count);
} else { formula = 0;
// console.log('sdfsdfs');
}
title = users[i].title.replace(/[^a-zа-яё0-9\s]/gi, ' ');
console.log('|',i+1,'|',users[i].author,'|','<a href="'+ users[i].url+'">' + title + '</a>','|', users[i].comments,'|', users[i].length,'|',users[i].percent,'|');
let z = Number(i + 1);
notfull = '\n|'+z+'|'+users[i].author+'|'+'<a href="'+ users[i].url+'">' + title + '</a>'+'|'+ users[i].comments+'|'+ users[i].length+'|\n';
const fs = require("fs");
fs.appendFileSync("..\\report"+d+".txt",notfull);
// console.log(notfull);
}
// console.log('|',i+1,'|',users[i].author,'|','<a href="'+ users[i].url+'">' + title + '</a>','|', users[i].comments,'|', users[i].length,'|');
// + '\n|'+i+1+'|'+users[i].author+'|'+'<a href="'+ users[i].url+'">' + title + '</a>'+'|'+ users[i].comments+'|'+ users[i].length+'|\n'
//console.log(notfull);
full = `\n
Подробности о проверках тут:
Details about checks are here:
https://steemit.com/hive-171319/@alexmove/just-another-script-update-the-uniqueness-check-is-even-more-automated
https://steemit.com/hive-171319/@alexmove/how-do-i-check-the-uniqueness-of-the-text-instruction-for-nodejs-scripts
https://steemit.com/hive-171319/@alexmove/proverka-unikalnosti-teksta-v-steemit-na-nodejs
Have a nice day!
`
const fs = require("fs");
fs.appendFileSync("..\\report"+d+".txt",full);
});
connection.end();
});
module.exports = router;
num
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
const mysql = require("mysql2");
const connection = mysql.createConnection({
host: "us-cdbr-east-05.cleardb.net",
user: "b9ce03f75c865a",
database: "heroku_cfdf255bd781813",
password: "503a3e51"
});
const sql = `UPDATE post_number SET number='0' LIMIT 1000`;
// const sql = `INSERT INTO post_number(id,number) VALUES('0','${post_number}')`;
console.log(sql);
connection.query(sql,function(err, results) {
if(err) console.log(err);
//console.log(results);
});
connection.end();
});
module.exports = router;
post
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
const mysql = require("mysql2");
const steem = require('steem');
const fs = require("fs");
let wifkey = 'xxxx';
let votey = "alexmove";
let num_day = 1;
//for (var yyy = 0; yyy < 2; yyy++)
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
d = d.replace(/-/gi, '');
let accounts = fs.readFileSync('..\\report'+d+'.txt').toString()
//fs.appendFileSync(__dirname+"\aaareport",'aaa');
//fs.appendFileSync("aaareport",'aaa');
//fs.appendFileSync("..\\aaareport",'aaa');
var currentPath = process.cwd();
console.log(currentPath);
console.log(__dirname+"aaareport");
// Prints: /Users/mjr
//console.log(path.dirname(__filename));
// Prints: /Users/mjr
//let accounts = fs.readFileSync('report20211225.txt').toString();
accounts = accounts.replace(/\n\n/gi, '\n');
console.log(accounts);
let permlink = new Date().toISOString().replace(/[^a-zA-Z0-9]+/g, '').toLowerCase();
console.log(permlink);
let title = "Test Mode Report Everyday. Steemit-Garden Checking The Uniqueness "+d;
steem.broadcast.comment(
wifkey, // Your posting wif
'', // Parent Author
'alexmove', // Parent Permlink
votey,
permlink, // Your post permlink
title, // Title
accounts, // Body
{ tags: ['steem','club5050','steemit','ukraine','js','games'], app: 'ganeshaway' }, // Json Metadata
function(err, result) {
console.log(err, result)});
});
module.exports = router;
set
var express = require('express');
var router = express.Router();
/* GET home page. */
router.get('/', function(req, res, next) {
res.render('index', { title: 'set' });
});
module.exports = router;
spisok
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
const fs = require("fs");
const steem = require('steem');
let body3R = '';
//https://steemit.com/hive-171319/@alexmove/the-word-game-post-2
var query = { limit : 100, tag : "hive-180821" };
//accounts.forEach(function (element,i,array){
steem.api.getDiscussionsByTrending(query, function(err, result) {
const mysql = require("mysql2");
const connection = mysql.createConnection({
host: "us-cdbr-east-05.cleardb.net",
user: "b9ce03f75c865a",
database: "heroku_cfdf255bd781813",
password: "503a3e51"
});
console.log(result.length);
if (result)
{
var i, len = result.length;
for (i = 0; i < len; i++)
{
var raw = result[i];
const sql = `INSERT INTO spisok(author) VALUES('${raw.author}')`;
connection.query(sql, function(err, results)
{
if(err) console.log(err);
console.log(results);
});
}
}
connection.end();
});
//});
});
module.exports = router;
unic
var express = require('express');
var router = express.Router();
/* GET users listing. */
router.get('/', function(req, res, next) {
res.send('unic');
const mysql = require("mysql2");
const steem = require('steem');
let wifkey = 'xxxxx';
let votey = "alexmove";
let num_day = 1;
//for (var yyy = 0; yyy < 2; yyy++)
{
//connection.on('error', function() {});
const connection = mysql.createConnection({
host: "us-cdbr-east-05.cleardb.net",
user: "b9ce03f75c865a",
database: "heroku_cfdf255bd781813",
password: "503a3e51"
});
let sql = `SELECT * FROM post_number` ;
connection.query(sql, function(err, results) {
if(err) console.log(err);
const users2 = results;
// console.log(results);
let post_number = users2[0].number;
// let post_number = 0;
console.log(post_number);
//console.log(sql);
/*
let sql_date = `SELECT * FROM date` ;
connection.query(sql_date, function(err, results) {
if(err) console.log(err);
const users2 = results;
// console.log(results);
let mytable_p = users2[0].p;
let mytable_c = users2[0].c;
let mytable_day = users2[0].a;
*/
var d = new Date(); // Today!
d.setDate(d.getDate()-num_day); // Yesterday!
d = d.toJSON().split("T")[0];
d = d.replace(/-/gi, '');
let mytable_p = 'p'+d;
// let mytable_p = 'p20211210';
//скрипт date
//});
//connection.end();
const axios = require('axios');
const FormData = require('form-data');
let users;
const sql = `SELECT * FROM ${mytable_p} ORDER BY length DESC`;
//const sql = `SELECT * FROM p0111 ORDER BY length DESC`;
// console.log(sql);
connection.query(sql, function(err, results) {
if(err) console.log(err);
// console.log(results);
users = results;
// console.log('[1]');
// console.log(results[results.length-1]);
// console.log(results.length);
// console.log(post_number);
//console.log(post_number);
console.log((users[post_number]));
//console.log(users[post_number-1]);
console.log (users[post_number].percent);
console.log (users[post_number].percent);
console.log (users[post_number].percent);
console.log (users[post_number].percent);
console.log (users[post_number].percent);
if ((users[post_number])&&(users[post_number].percent===0))
// if (users[post_number])
{
console.log(results.length);
// console.log(results.length);
// console.log(results.length);
// console.log(results.length);
// console.log(post_number);
// console.log('[2]');
// console.log(results[77]);
/// console.log(results[4]);
// console.log(post_number);
// console.log(users[post_number].author);
let data;
async function makeGetRequest(qqq,aaa) {
const form_data = new FormData();
// form_data.append('text', qqq);
form_data.append('url', qqq);
// form_data.append('test', '1');
let res = await axios.post('https://content-watch.ru/public/api/?action=CHECK_URL&key=lsjhl80J7RiB4FU', form_data,
{ headers: form_data.getHeaders() });
data = res.data;
console.log(data);
console.log(data.percent);
/*
function highlight_words(text, hightlight)
{
var words = text.split(" ");
for (var i = 0; i < hightlight.length; i++)
{
words[ hightlight[i][0] ] = '<strong>' + words[ hightlight[i][0] ];
words[ hightlight[i][1] ] = words[ hightlight[i][1] ] + '</strong>';
}
return words.join(" ");
}
*/
// console.log(data.matches);
console.log(data.matches[3]);
let answer2 = data.percent + "% уникальности текста (uniqueness of the text) " + " <br> " + "Здравствуйте. Это полуавтоматическая проверка уникальности сервисом content-watch. Пожалуйста, сообщайте о неточностях. <b>Уникальность вашей статьи согласно сервиса content-watch.ru равна: " + data.percent + "% </b><br><br>Hello. This is a half-automatic uniqueness check by the content-watch service Please report any inaccuracies <b> The uniqueness of your stat th according to the content-watch.ru service is equal to: "+ data.percent +"% </b><br> ";
var d = new Date(); // Today!
d.setDate(d.getDate() - 1); // Yesterday!
d = d.toJSON().split("T")[0];
let permlink = new Date().toISOString().replace(/[^a-zA-Z0-9]+/g, '').toLowerCase();
const url = new URL(qqq);
let test2 = url.pathname.split('/');
test2 = test2[3];
console.log('TITLE');
console.log(test2);
console.log(qqq);
console.log('TITLE');
//const params = new URLSearchParams(url.search)
//console.log(params.get("filter"))
steem.broadcast.comment(
wifkey, // Your posting wif
aaa, // Parent Author
test2, // Parent Permlink
votey,
permlink, // Your post permlink
'', // Title
answer2, // Body
{ tags: ['thanks'], app: 'ganeshaway' }, // Json Metadata
function(err, result) {
console.log(err, result)});
const sql = `UPDATE ${mytable_p} SET percent='${data.percent}' WHERE url='${users[post_number].url}' LIMIT 1000`;
console.log(sql)
connection.query(sql,function(err, results) {
if(err) console.log(err);
// console.log(results);
});
// if ((users[post_number])&&(users[percent]===0))
console.log(post_number);
console.log(post_number);
console.log(post_number);
console.log(post_number);
console.log(post_number);
console.log(post_number);
post_number++;
const sql8 = `UPDATE post_number SET number='${post_number}' LIMIT 1000`;
// post_number=post_number-1;
connection.query(sql8,function(err, results) {
if(err) console.log(err);
console.log(results);
});
}
const url = new URL('http://www.somedomain.com/account/search?filter=a#top');
console.log(url.pathname.split('/'));
const params = new URLSearchParams(url.search)
//console.log(params.get("filter"))
// console.log('makeGetRequest')
makeGetRequest('https://steemit.com'+users[post_number].url,users[post_number].author);
// console.log(users[post_number].url)
// console.log(users[post_number].author)
}
else
{
// let post_number = 0;
console.log(post_number);
post_number++;
const sql8 = `UPDATE post_number SET number='${post_number}' LIMIT 1000`;
// post_number=post_number-1;
//console.log(sql);
connection.query(sql8,function(err, results) {
if(err) console.log(err);
// console.log(results);
});
}
});
});
// connection.end();
}
});
module.exports = router;
Have a nice day!
Congratulations, your post has been upvoted by @scilwa, which is a curating account for @R2cornell's Discord Community. We can also be found on our hive community & peakd as well as on my Discord Server
Felicitaciones, su publication ha sido votado por @scilwa. También puedo ser encontrado en nuestra comunidad de colmena y Peakd así como en mi servidor de discordia