import hashlib
import sys
import os
import time
import random
import re
from lxml import html
from requests_html import HTMLSession
cache_dir = 'cache'
abspath = os.path.abspath(__file__)
dir_name = os.path.dirname(abspath)
os.chdir(dir_name)
session = HTMLSession()
user_stat = {}
clean_re = re.compile('[^а-яА-Я]')
def random_sleep(func):
def slp(*args, **kwargs):
tm = random.randint(500, 1500) / 1000
time.sleep(tm)
return func(*args, **kwargs)
return slp
@random_sleep
def req(url):
print(f'Fetch: {url}')
return session.get(url)
if __name__ == '__main__':
path = None
max_page = 1
min_page = 1
if not os.path.exists(cache_dir):
os.mkdir(cache_dir)
if len(sys.argv) > 3:
min_page = int(sys.argv[3])
max_page = int(sys.argv[2])
path = sys.argv[1]
elif len(sys.argv) > 2:
max_page = int(sys.argv[2])
path = sys.argv[1]
elif len(sys.argv) > 1:
path = sys.argv[1]
if path:
for p in range(min_page, max_page + 1):
lnk = path.format(page=p)
hash_sum = hashlib.sha512(lnk.encode()).hexdigest()
fn = os.path.join(cache_dir, hash_sum) + '.html'
if os.path.exists(fn):
with open(fn, 'rt') as fp:
r = fp.read()
else:
r = req(lnk).text
with open(fn, 'wt') as fp:
fp.write(r)
root = html.fromstring(r)
for article in root.xpath('//article[contains(@class, "message--post")]'):
user = ''.join(article.xpath('.//h4[contains(@class, "message-name")]/a[contains(@class, "username")]/text()')).strip()
text = ''.join(article.xpath('.//div[contains(@class, "bbWrapper")]/text()')).strip()
u = user
if u not in user_stat:
user_stat[u] = {
'posts': 0,
'words': 0,
'uniq': set(),
'post_length': [],
}
user_stat[u]['posts'] += 1
word_list = text.split()
word_length = len(word_list)
if word_length > 0:
user_stat[u]['words'] += len(word_list)
user_stat[u]['post_length'].append(len(word_list))
for w in word_list:
user_stat[u]['uniq'].add(w)
stats = sorted(user_stat.items(), key=lambda e: e[1]['posts'], reverse=True)
total_posts = 0;
with open('bbcode.txt', 'wt') as fp:
fp.write('[TABLE]')
fp.write('[TR][TD]user[/TD][TD]posts[/TD][TD]words[/TD][TD]avg post length[/TD][TD]shortest post[/TD][TD]longest post[/TD][TD]uniq words[/TD][TD]uniq ratio[/TD][/TR]')
for u, stat in stats:
uniq = len(stat["uniq"])
ratio = None
avg_words_in_post = None
min_post = None
max_post = None
if uniq > 0:
ratio = round(stat["words"]/uniq, 2)
if len(stat['post_length']) > 0:
avg_words_in_post = round(sum(stat['post_length']) / len(stat['post_length']))
min_post = min(stat['post_length'])
max_post = max(stat['post_length'])
total_posts += stat["posts"]
fp.write('[TR]')
fp.write(f'[TD]{u}[/TD][TD]{stat["posts"]}[/TD][TD]{stat["words"]}[/TD][TD]{avg_words_in_post}[/TD][TD]{min_post}[/TD][TD]{max_post}[/TD][TD]{uniq}[/TD][TD]{ratio}[/TD]')
fp.write('[/TR]')
fp.write('[/TABLE]')
print(total_posts)
