281 lines
8.7 KiB
Python
281 lines
8.7 KiB
Python
import os
|
||
import re
|
||
import fitz
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
|
||
ALL_DIR = Path("/home/matrixhasyou/mutt/exported_emails/all")
|
||
OUTPUT_HTML = Path("/home/matrixhasyou/mutt/chronology.html")
|
||
|
||
def parse_pdf(filepath):
|
||
try:
|
||
doc = fitz.open(filepath)
|
||
text = ""
|
||
for page_num in range(min(5, len(doc))):
|
||
text += doc[page_num].get_text()
|
||
doc.close()
|
||
return text
|
||
except Exception as e:
|
||
return ""
|
||
|
||
def is_valid_reply(text):
|
||
text_lower = text.lower()
|
||
formal_markers = [
|
||
'ваше обращение рассмотрено', 'сообщаем вам', 'уведомляем',
|
||
'в соответствии с', 'в пределах компетенции',
|
||
'направляем в ваш адрес', 'в рамках действующего',
|
||
'разъясняем', 'доводим до вашего сведения',
|
||
'отказано', 'рассмотрев ваше', 'информация предоставляется'
|
||
]
|
||
substantive_markers = [
|
||
'удовлетворено', 'приняты меры', 'выявлены нарушения',
|
||
'обязать устранить', 'проведена проверка', 'выдано предписание',
|
||
'штраф наложен', 'материалы переданы', 'перерасчет', 'исполнено'
|
||
]
|
||
indicators = ['ответ на обращение', 'рассмотрение', 'сообщаем', 'уведомляем', 'обращение', 'запрос']
|
||
|
||
if any(kw in text_lower for kw in formal_markers + substantive_markers + indicators):
|
||
return True
|
||
return False
|
||
|
||
def extract_meta_from_text(text):
|
||
date_match = re.search(r'Date:\s*(.*?)\n', text)
|
||
from_match = re.search(r'From:\s*(.*?)\n', text)
|
||
to_match = re.search(r'To:\s*(.*?)\n', text)
|
||
subject_match = re.search(r'Subject:\s*(.*?)\n', text)
|
||
|
||
return {
|
||
'date': date_match.group(1).strip() if date_match else "",
|
||
'from': from_match.group(1).strip() if from_match else "",
|
||
'to': to_match.group(1).strip() if to_match else "",
|
||
'subject': subject_match.group(1).strip() if subject_match else ""
|
||
}
|
||
|
||
def clean_text_for_html(text):
|
||
# Убираем технические заголовки из начала текста, если они есть
|
||
lines = text.split('\n')
|
||
cleaned = []
|
||
in_headers = True
|
||
for line in lines:
|
||
if in_headers and (line.startswith('From:') or line.startswith('To:') or line.startswith('Date:') or line.startswith('Subject:') or line.startswith('Message-ID:')):
|
||
continue
|
||
if line.strip() == '':
|
||
if in_headers: continue
|
||
in_headers = False
|
||
cleaned.append(line)
|
||
|
||
res = "\n".join(cleaned).strip()
|
||
# Ограничиваем длину текста
|
||
if len(res) > 2000:
|
||
res = res[:2000] + "...\n[ТЕКСТ СОКРАЩЕН]"
|
||
|
||
# Escape HTML
|
||
res = res.replace("&", "&").replace("<", "<").replace(">", ">")
|
||
# Convert newlines to br
|
||
res = res.replace("\n", "<br>")
|
||
return res
|
||
|
||
all_emails = []
|
||
stats = {'sent': 0, 'replies': 0, 'skipped': 0}
|
||
|
||
print("Parsing emails...")
|
||
for f in ALL_DIR.glob("*.pdf"):
|
||
parts = f.stem.split('_', 3)
|
||
if len(parts) < 4: continue
|
||
|
||
domain = parts[0]
|
||
date_str = parts[1]
|
||
sender = parts[2]
|
||
subject = parts[3]
|
||
|
||
if sender.lower() in ['no-reply', 'noreply', 'notification']:
|
||
stats['skipped'] += 1
|
||
continue
|
||
|
||
text = parse_pdf(f)
|
||
if not text.strip(): continue
|
||
|
||
meta = extract_meta_from_text(text)
|
||
|
||
is_sent = False
|
||
if sender == 'ktybdsq' or 'ktybdsq@gmail.com' in meta['from'].lower():
|
||
is_sent = True
|
||
|
||
if is_sent:
|
||
stats['sent'] += 1
|
||
all_emails.append({'file': f.name, 'date': date_str, 'subject': subject, 'text': text, 'meta': meta, 'is_sent': True})
|
||
else:
|
||
if is_valid_reply(text):
|
||
stats['replies'] += 1
|
||
all_emails.append({'file': f.name, 'date': date_str, 'subject': subject, 'text': text, 'meta': meta, 'is_sent': False})
|
||
else:
|
||
stats['skipped'] += 1
|
||
|
||
print(f"Stats: {stats}")
|
||
all_emails.sort(key=lambda x: x['date'])
|
||
|
||
# Generate HTML
|
||
html = """<!DOCTYPE html>
|
||
<html lang="ru">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>Хронология Переписки | LEGAL SYSTEM</title>
|
||
<style>
|
||
body {
|
||
margin: 0;
|
||
padding: 20px;
|
||
background: #000;
|
||
color: #00ff00;
|
||
font-family: 'Courier New', Courier, monospace;
|
||
}
|
||
|
||
.header {
|
||
text-align: center;
|
||
margin-bottom: 40px;
|
||
}
|
||
|
||
.header h1 {
|
||
text-shadow: 0 0 10px #00ff00;
|
||
}
|
||
|
||
.btn-back {
|
||
color: #00ff00;
|
||
text-decoration: none;
|
||
border: 1px solid #00ff00;
|
||
padding: 5px 15px;
|
||
display: inline-block;
|
||
margin-bottom: 20px;
|
||
}
|
||
.btn-back:hover {
|
||
background: #00ff00;
|
||
color: #000;
|
||
box-shadow: 0 0 10px #00ff00;
|
||
}
|
||
|
||
.stats {
|
||
text-align: center;
|
||
margin-bottom: 30px;
|
||
padding: 10px;
|
||
border: 1px dashed #008800;
|
||
}
|
||
|
||
.timeline {
|
||
display: flex;
|
||
flex-direction: column;
|
||
gap: 20px;
|
||
max-width: 1000px;
|
||
margin: 0 auto;
|
||
}
|
||
|
||
.message {
|
||
padding: 15px;
|
||
border: 1px solid #00ff00;
|
||
width: 70%;
|
||
position: relative;
|
||
background: rgba(0, 20, 0, 0.8);
|
||
box-shadow: 0 0 10px rgba(0, 255, 0, 0.2);
|
||
}
|
||
|
||
.message.sent {
|
||
align-self: flex-start;
|
||
border-left: 5px solid #00ff00;
|
||
}
|
||
|
||
.message.received {
|
||
align-self: flex-end;
|
||
border-right: 5px solid #ffcc00;
|
||
border-color: #ffcc00;
|
||
color: #ffcc00;
|
||
}
|
||
|
||
.msg-header {
|
||
font-size: 0.9rem;
|
||
margin-bottom: 10px;
|
||
border-bottom: 1px dashed currentcolor;
|
||
padding-bottom: 5px;
|
||
}
|
||
|
||
.msg-title {
|
||
font-weight: bold;
|
||
margin-bottom: 10px;
|
||
font-size: 1.1rem;
|
||
}
|
||
|
||
.msg-body {
|
||
font-size: 0.95rem;
|
||
line-height: 1.4;
|
||
max-height: 300px;
|
||
overflow-y: auto;
|
||
}
|
||
|
||
.msg-body::-webkit-scrollbar {
|
||
width: 8px;
|
||
}
|
||
.msg-body::-webkit-scrollbar-track {
|
||
background: #000;
|
||
}
|
||
.msg-body::-webkit-scrollbar-thumb {
|
||
background: currentcolor;
|
||
}
|
||
|
||
.msg-date {
|
||
position: absolute;
|
||
top: -10px;
|
||
background: #000;
|
||
padding: 0 5px;
|
||
font-size: 0.8rem;
|
||
}
|
||
.sent .msg-date { left: 10px; }
|
||
.received .msg-date { right: 10px; }
|
||
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<a href="index.html" class="btn-back">[ Назад в меню ]</a>
|
||
|
||
<div class="header">
|
||
<h1>PROJECT: LEGAL CHRONOLOGY<span style="animation: blink 1s infinite;">_</span></h1>
|
||
<p>Архив юридически значимой переписки с контролирующими органами.</p>
|
||
</div>
|
||
|
||
<div class="stats">
|
||
SYS_STATS: [ОТПРАВЛЕНО: {sent}] | [ПОЛУЧЕНО ОТВЕТОВ: {replies}] | [ОТСЕЯНО МУСОРА: {skipped}]
|
||
</div>
|
||
|
||
<div class="timeline">
|
||
"""
|
||
|
||
html = html.replace("{sent}", str(stats['sent'])).replace("{replies}", str(stats['replies'])).replace("{skipped}", str(stats['skipped']))
|
||
|
||
for msg in all_emails:
|
||
cls = "sent" if msg['is_sent'] else "received"
|
||
sender_str = msg['meta']['from'] or msg['file'].split('_')[2]
|
||
receiver_str = msg['meta']['to'] or "Госорганы / УК" if msg['is_sent'] else "MatrixHasYou"
|
||
|
||
subj = msg['subject']
|
||
if not subj and msg['meta']['subject']:
|
||
subj = msg['meta']['subject']
|
||
|
||
body = clean_text_for_html(msg['text'])
|
||
|
||
html += f"""
|
||
<div class="message {cls}">
|
||
<div class="msg-date">{msg['date']}</div>
|
||
<div class="msg-header">
|
||
<div><strong>ОТ:</strong> {sender_str}</div>
|
||
<div><strong>КОМУ:</strong> {receiver_str}</div>
|
||
</div>
|
||
<div class="msg-title">ТЕМА: {subj}</div>
|
||
<div class="msg-body">{body}</div>
|
||
</div>
|
||
"""
|
||
|
||
html += """
|
||
</div>
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
OUTPUT_HTML.write_text(html, encoding='utf-8')
|
||
print(f"Saved HTML to {OUTPUT_HTML}")
|