Initial commit

This commit is contained in:
Анатолий Богомолов 2024-01-31 19:37:01 +10:00
parent fd28449133
commit d21542a9ad
44 changed files with 1624 additions and 0 deletions

4
.gitignore vendored
View File

@ -160,3 +160,7 @@ cython_debug/
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
#.idea/
*.db
*.session
.vscode/settings.json
*session_journal

21
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,21 @@
{
"configurations": [
{
"name": "Python: Flask",
"type": "python",
"request": "launch",
"module": "flask",
"env": {
"FLASK_APP": "app",
"FLASK_DEBUG": "1"
},
"args": [
"run",
"--no-debugger",
"--no-reload",
],
"jinja": true,
"justMyCode": true
}
]
}

11
Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM python:3.11
COPY app/ /app/
COPY migrations/ /migrations/
COPY paper /paper/
COPY config.py config.py
COPY make_celery.py make_celery.py
COPY requirements.txt requirements.txt
RUN pip install -r requirements.txt

20
app/__init__.py Normal file
View File

@ -0,0 +1,20 @@
from flask import Flask
from config import Config
from app.blueprints import blueprints
from app.extensions import db, migrate
from app.celery import celery_init_app
def create_app(config_class=Config):
app = Flask(__name__)
app.config.from_object(config_class)
db.init_app(app)
migrate.init_app(app)
for blueprint in blueprints:
app.register_blueprint(blueprint)
celery_init_app(app)
return app

View File

@ -0,0 +1,7 @@
from .api.routes import api
from .frontend.routes import frontend
blueprints = (
api,
frontend,
)

View File

@ -0,0 +1,11 @@
from flask import Blueprint
from .tasks.routes import tasks
from .sessions.routes import sessions
from .users.routes import users
api = Blueprint("api", __name__, url_prefix="/api")
api.register_blueprint(users)
api.register_blueprint(tasks)
api.register_blueprint(sessions)

View File

@ -0,0 +1,81 @@
from flask import Blueprint, Response, jsonify, request
from app.extensions import db
from app.models.session import Session
from app.models.task import Task
from paper.errors import NeedPasswordException
from paper.parser import PaperParser
sessions = Blueprint("sessions", __name__, url_prefix="/sessions")
@sessions.route("/", methods=["POST"])
async def create_session(**kwargs):
data = kwargs or request.json or request.form
data["name"] = data["name"].replace("/", "").replace("\\", "")
session = Session.query.filter_by(name=data.get("name")).first()
if not session:
session = Session(
name=data.get("name"),
authorized=False
)
db.session.add(session)
response = {"status": "ok"}
async with PaperParser(data.get("name")) as parser:
try:
result = await parser.sign_in(**data)
if hasattr(result, "phone_code_hash"):
response = {
"status": "error",
"message": "Need secret code from telegram",
"action": "code",
"phone_hash": result.phone_code_hash,
}
else:
session.authorized = True
except NeedPasswordException:
response = {
"status": "error",
"message": "Need 2FA Password",
"action": "password",
}
except Exception as e:
response = {
"status": "exception",
"message": str(e),
}
finally:
db.session.commit()
return jsonify(response)
@sessions.route("/", methods=["GET"])
def get_sessions(**kwargs):
return Session.query.filter_by(authorized=True).all()
@sessions.route("/<int:id>", methods=["DELETE"])
async def remove_session(id: int):
session: Session = Session.query.get_or_404(id)
if session.has_depending_tasks():
return 'Есть зависимые задачи', 409
async with PaperParser(session.name) as parser:
try:
await parser.client.log_out()
except Exception as e:
return Response(e, 500)
db.session.delete(session)
db.session.commit()
return Response('ok', status=200)

View File

@ -0,0 +1,27 @@
from app.extensions import db
from app.models.task import Task
def success_state(task_id):
task = Task.query.filter_by(task_id=task_id).one()
task.status = "SUCCESS"
task.status_message = "Задача выполнена успешно"
db.session.commit()
def failure_state(task_id, exception):
task = Task.query.filter_by(task_id=task_id).one()
task.status = "FAILURE"
task.status_message = f"Произошла ошибка: {str(exception)}"
db.session.commit()
def run_state(task_id, task_record_id):
task = Task.query.get(task_record_id)
task.task_id = task_id
task.status = "RUNNING"
task.status_message = "Задача запущена"
db.session.commit()

View File

@ -0,0 +1,83 @@
import uuid
from flask import Blueprint, render_template, request
from app.extensions import db
from app.models.task import Task
from app.models.session import Session
from .tasks import send_messages_task, add_to_group_task
tasks = Blueprint("tasks", __name__, url_prefix="/tasks", template_folder="templates")
@tasks.route("/", methods=["GET"])
def get_tasks():
tasks = Task.query.all()
return render_template("tasks_cards.j2", tasks=tasks)
@tasks.route("/<int:task_id>", methods=["GET"])
def get_task(task_id: int):
...
@tasks.route("/session/<int:session_id>", methods=["POST"])
def create_task(session_id: int):
info = request.form
session = Session.query.get_or_404(session_id)
task = Task(
name = str(uuid.uuid4()),
session = session,
status = "CREATED",
status_message = "Задача создана",
type = info.get('task'),
url = info.get('url'),
message = info.get('message'),
file = None
)
db.session.add(task)
db.session.commit()
start_task(task.id)
return 'Created', 200
@tasks.route("/<int:task_id>", methods=["DELETE"])
def delete_task(task_id: int):
task: Task = Task.query.get_or_404(task_id)
stop_task(task.id)
db.session.delete(task)
db.session.commit()
return 'Deleted', 200
@tasks.route("/<int:task_id>/stop", methods=["PUT"])
def stop_task(task_id: int):
task = Task.query.get_or_404(task_id)
fn = None
match task.type:
case 'message':
fn = send_messages_task
case 'add':
fn = add_to_group_task
if task.task_id:
task_fn = fn.AsyncResult(task.task_id)
task_fn.abort()
return 'Stopped', 200
@tasks.route("/<int:task_id>/start", methods=["PUT"])
def start_task(task_id: int):
task: Task = Task.query.get_or_404(task_id)
match task.type:
case 'message':
send_messages_task.delay(task_id=task_id)
case 'add':
add_to_group_task.delay(task_id=task_id)
return 'Started', 200

View File

@ -0,0 +1,47 @@
import time
from celery import shared_task
import asyncio
from paper.parser import PaperParser
from app.models.task import Task
from app.extensions import db
from .handlers import run_state, failure_state, success_state
async def add_to_group(session, task, task_self):
async with PaperParser(session.name) as parser:
await parser.invite_users(session.users, task.url, task_self)
async def sending_message(session, task, task_self):
async with PaperParser(session.name) as parser:
await parser.send_messages(session.users, task.message, task.file, task_self)
@shared_task(bind=True)
def add_to_group_task(self, task_id):
try:
task: Task = Task.query.get(task_id)
run_state(self.request.id, task_id)
session = task.session
time.sleep(10)
asyncio.run(
add_to_group(session, task, self)
)
except Exception as e:
failure_state(self.request.id, e)
raise e
else:
success_state(self.request.id)
@shared_task(bind=True)
def send_messages_task(self, task_id):
task: Task = Task.query.get(task_id)
session = task.session
users = session.users

View File

@ -0,0 +1,34 @@
{% for task in tasks %}
<div data-task="{{ task.id }}" class="col mb-3">
<div class="card">
<div class="card-body">
<h3 class="card-title">{{ task.name }}</h3>
<ul class="list-group list-group-flush">
<li class="list-group-item">Статус: {{ task.status }}</li>
<li class="list-group-item">Сообщение: {{ task.status_message }}</li>
<li class="list-group-item">Действие: {{ task.type }}</li>
<li class="list-group-item">Сессия: {{ task.session.name }}</li>
</ul>
</div>
<div class="card-body">
<button
hx-delete="/api/tasks/{{task.id}}"
hx-swap="delete"
hx-target='[data-task="{{ task.id }}"]'
hx-confirm="Вы уверены, что хотите удалить эту задачу?"
class="btn btn-outline-danger mt-2 mx-2"
>
Удалить
</button>
<button hx-put="/api/tasks/{{task.id}}/stop" hx-swap="none" class="btn btn-outline-secondary mt-2 mx-2">
Стоп
</button>
<button hx-put="/api/tasks/{{task.id}}/start" hx-swap="none" class="btn btn-outline-success mt-2 mx-2">
Запустить
</button>
</div>
</div>
</div>
{% else %}
<small id="helpId" class="form-text">Здесь ничего нет.</small>
{% endfor %}

View File

@ -0,0 +1,58 @@
from flask import Blueprint, render_template, request
from app.models.session import Session
from app.models.user import User
from app.extensions import db
from paper.parser import PaperParser
users = Blueprint("users", __name__, url_prefix="/users", template_folder="templates")
@users.route("/parse/<int:session_id>", methods=["POST"])
async def parse_users(session_id: int):
session = Session.query.get_or_404(session_id)
data = request.form
async with PaperParser(session.name) as parser:
users = await parser.get_participants(data.get("group"))
for user in users:
if not user.username:
continue
exists = db.session.query(User.id).filter_by(username=user.username).first()
if not exists:
db.session.add(
User(
first_name=user.first_name,
last_name=user.last_name,
username=user.username,
phone=user.phone,
session=session,
)
)
db.session.commit()
return render_template("user_cards.j2", users=session.users)
@users.route("/<int:session_id>", methods=["GET"])
def get_users(session_id: int):
users = Session.query.get_or_404(session_id).users
return render_template("user_cards.j2", users=users)
@users.route("/<int:session_id>", methods=["DELETE"])
def delete_users(session_id):
session: Session = Session.query.get_or_404(session_id)
if session.has_depending_tasks():
return 'Есть зависимые задачи', 409
User.query.filter_by(session=session).delete()
db.session.commit()
return render_template("user_cards.j2", users=[])

View File

@ -0,0 +1,18 @@
{% for user in users %}
<div class="col mb-3">
<div class="card h-100">
<div class="card-body">
<h3 class="card-title">{{ user.first_name }}</h3>
</div>
<ul class="list-group list-group-flush">
<li class="list-group-item">{{ user.first_name }}</li>
<li class="list-group-item">{{ user.last_name }}</li>
<li class="list-group-item"><a href="https://t.me/{{ user.username }}" target=”_blank”>Open chat with @{{
user.username }}</a></li>
<li class="list-group-item"><a href="tel:{{ user.phone }}">{{ user.phone }}</a></li>
</ul>
</div>
</div>
{% else %}
<small id="helpId" class="form-text">Здесь ничего нет.</small>
{% endfor %}

View File

@ -0,0 +1,23 @@
from flask import Blueprint, render_template
from app.blueprints.api.sessions.routes import get_sessions
from app.blueprints.api.users.routes import get_users
from app.blueprints.api.tasks.routes import get_tasks
frontend = Blueprint("frontend", __name__, url_prefix="/", template_folder="templates", static_folder="static", static_url_path="/static/frontend")
@frontend.route("/")
def index():
sessions = get_sessions()
return render_template("index.j2", sessions=sessions)
@frontend.route("/add")
def add():
return render_template("add.j2")
@frontend.route("/tasks")
def tasks():
return render_template("tasks.j2", tasks_template=get_tasks())
@frontend.route("/parse/<int:id>")
def parse(id: int):
return render_template("parse.j2", session_id=id, users_template=get_users(id))

View File

@ -0,0 +1,115 @@
const API_URL = window.location.origin;
const nameInput = document.querySelector("#input-name");
const phoneInput = document.querySelector("#input-phone");
const addSessionForm = document.querySelector("#add-session-form");
const openFileButton = document.querySelector("#open-file-btn");
const secretCodeModal = new bootstrap.Modal('#secretCodeModal');
const secretCodeForm = document.querySelector("#secret-code-form");
const secretCodeInput = document.querySelector("#input-secret-code");
const passwordModal = new bootstrap.Modal('#passwordModal');
const passwordForm = document.querySelector("#password-form");
const passwordInput = document.querySelector("#input-password");
let phoneHash = "";
const handleCreation = async (e) => {
e.preventDefault();
if (phoneInput.value == "" || nameInput.value == "") {
alert("Пожалуйста, заполните все поля.");
return;
}
const data = getAuthData();
res = await fetch(`${API_URL}/api/sessions/`, {
method: "POST",
body: JSON.stringify(data),
headers: {
"Content-Type": "application/json",
}
});
result = await res.json();
switch(result.status) {
case "ok":
window.location = "/";
break;
case "error":
handleError(result);
break;
case "exception":
alert(`Ошибка: ${result.message}.`);
break;
default:
alert(`Неизвестная ошибка. Проверьте логи`);
}
}
const getAuthData = () => {
const inputFields = [passwordInput, secretCodeInput, nameInput, phoneInput]
data = {}
inputFields.forEach((el) => {
if (el.value !== "") {
data[el.attributes.name.value] = el.value.replace(/ /g, '');
}
})
if (phoneHash !== "")
data["phone_hash"] = phoneHash;
return data;
}
const handleRequest = async (fn) => {
}
const handleError = (result) => {
switch(result.action) {
case "code":
phoneHash = result.phone_hash;
secretCodeModal.show();
break;
case "password":
passwordModal.show();
break;
}
}
addSessionForm.addEventListener("submit", handleCreation);
secretCodeForm.addEventListener("submit", handleCreation);
passwordForm.addEventListener("submit", handleCreation);
// openFileButton.addEventListener("click", handleOpening);
function handleOpening() {
let input = document.createElement('input');
input.type = 'file';
input.accept = ".zip";
input.onchange = async () => {
// you can use this method to get file and perform respective operations
let archive = Array.from(input.files)[0];
let formData = new FormData();
formData.append("archive", archive);
response = await fetch(`${API_URL}/api/save_session_file.php`, { method: "POST", body: formData });
if (response.status == "201") {
window.location = "/"
} else {
const error_text = await response.text();
const error_status = await response.status;
alert(`${error_status}: ${error_text}`);
}
};
input.click();
}

View File

@ -0,0 +1,105 @@
{% extends "base.j2" %}
{% block title %}
PaperParser: Вход в аккаунт
{% endblock title %}
{% block main %}
<div class="container ">
<!-- Modal Body -->
<!-- if you want to close by clicking outside the modal, delete the last endpoint:data-bs-backdrop and data-bs-keyboard -->
<div class="modal fade" id="secretCodeModal" tabindex="-1" data-bs-backdrop="static" data-bs-keyboard="false"
role="dialog" aria-labelledby="modalTitleId" aria-hidden="true">
<div class="modal-dialog modal-dialog-scrollable modal-dialog-centered modal-lg" role="document">
<div class="modal-content">
<form action="" id="secret-code-form">
<div class="modal-header">
<h5 class="modal-title" id="modalTitleId">Введите секретный код</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<div class="mb-3">
<div class="form-floating mb-3">
<input type="text" name="code" id="input-secret-code" class="form-control"
placeholder="Секретный код">
<label for="input-secret-code">Секретный код</label>
</div>
<small id="helpId" class="form-text">
Вы должны были получить 6-ти значный код в вашем
аккаунте. Если это не так, обратитесь к администратору, проверьте логи и вывод прокси
сервера.
</small>
</div>
</div>
<div class="modal-footer">
<button type="submit" class="btn btn-primary">Подтвердить</button>
</div>
</form>
</div>
</div>
</div>
<div class="modal fade" id="passwordModal" tabindex="-1" data-bs-backdrop="static" data-bs-keyboard="false"
role="dialog" aria-labelledby="modalTitleId" aria-hidden="true">
<div class="modal-dialog modal-dialog-scrollable modal-dialog-centered modal-lg" role="document">
<div class="modal-content">
<form action="" id="password-form">
<div class="modal-header">
<h5 class="modal-title" id="modalTitleId">Введите пароль</h5>
<button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
</div>
<div class="modal-body">
<div class="mb-3">
<div class="form-floating mb-3">
<input type="text" name="password" id="input-password" class="form-control"
placeholder="Пароль">
<label for="input-password">Пароль</label>
</div>
<small id="helpId" class="form-text">Введите пароль двухфакторной аутентификации. Если у акаунта его нет, да поможет вам бог.</small>
</div>
</div>
<div class="modal-footer">
<button type="submit" class="btn btn-primary">Подтвердить</button>
</div>
</form>
</div>
</div>
</div>
<form action="" id="add-session-form">
<!-- Telegram Account Info -->
<div class="mb-3">
<label for="" class="form-label">Данные телеграм аккаунта</label>
<div class="row row-cols-1 row-cols-sm-2">
<div class="col">
<div class="form-floating mb-3">
<input type="text" name="name" id="input-name" class="form-control" placeholder="Имя">
<label for="input-name">название сессии</label>
</div>
</div>
<div class="col">
<div class="form-floating mb-3">
<input type="phone" name="phone" id="input-phone" class="form-control" placeholder="Phone">
<label for="input-phone">Номер телефона</label>
</div>
</div>
</div>
</div>
<button type="submit" class="btn btn-primary">Отправить код</button>
</form>
<hr>
<form action="/api/sessions/file" method="POST">
<label for="open-file-btn" class="form-label">Архив с аккаунтом (или .session файл)</label>
<div class="mb-3">
<button id="open-file-btn" type="submit" class="btn btn-secondary mt-3 disabled">Открыть файл</button>
</div>
</form>
</div>
{% endblock main %}
{% block scripts %}
<script src="{{ url_for('static', filename='frontend/add.js') }}"></script>
{% endblock scripts %}

View File

@ -0,0 +1,64 @@
<!doctype html>
<html lang="en">
<head>
<title>{% block title %}{% endblock %}</title>
<!-- Required meta tags -->
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<!-- Bootstrap CSS v5.2.1 -->
<link href="https://cdn.jsdelivr.net/npm/bootstrap@5.2.1/dist/css/bootstrap.min.css" rel="stylesheet"
integrity="sha384-iYQeCzEYFbKjA/T2uDLTpkwGzCiq6soy8tYaI1GyVh/UjpbCx/TYkiZhlZB6+fzT" crossorigin="anonymous">
<link rel="shortcut icon" href="{{ url_for('static', filename='favicon.ico') }}">
</head>
<body class="d-flex flex-column min-vh-100">
<header>
<!-- place navbar here -->
<nav class="navbar navbar-expand-sm navbar-light bg-light mb-3">
<div class="container justify-content-beetwen">
<a class="navbar-brand" href="/">PaperParser</a>
<ul class="nav">
<li><a class="nav-link" href="/tasks">Задачи</a></li>
<li><a class="btn btn-outline-success" href="/add" role="button">Войти</a></li>
</ul>
</div>
</nav>
</header>
<main>
{% block main %}
{% endblock %}
</main>
<footer class="container mt-auto">
<div class="d-flex flex-wrap justify-content-between align-items-center py-3 my-4 border-top">
<p class="col-md-4 mb-0 text-body-secondary">Anatoly "wineT" Bogomolov</p>
<ul class="nav col-md-4 justify-content-end">
<li class="nav-item"><a href="https://t.me/AnatolyFL" class="nav-link px-2 text-body-secondary">Telegram</a></li>
<li class="nav-item"><a href="https://mastodon.ml/@winet" class="nav-link px-2 text-body-secondary">Mastodon</a></li>
<li class="nav-item"><a href="#" class="nav-link px-2 text-body-secondary">Project on Gitiea</a></li>
</ul>
</div>
</footer>
<!-- Bootstrap JavaScript Libraries -->
<script src="https://cdn.jsdelivr.net/npm/@popperjs/core@2.11.6/dist/umd/popper.min.js"
integrity="sha384-oBqDVmMz9ATKxIep9tiCxS/Z9fNfEXiDAYTujMAeBAsjFuCZSmKbSSUnQlmh/jp3" crossorigin="anonymous">
</script>
<script src="https://cdn.jsdelivr.net/npm/bootstrap@5.2.1/dist/js/bootstrap.min.js"
integrity="sha384-7VPbUDkoPSGFnVtYi0QogXtr74QeVeeIs99Qfg5YCF+TidwNdjvaKZX19NZ/e6oz" crossorigin="anonymous">
</script>
<script src="https://unpkg.com/htmx.org@1.9.10" integrity="sha384-D1Kt99CQMDuVetoL1lrYwg5t+9QdHe7NLX/SoJYkXDFfX37iInKRy5xLSi8nO7UC" crossorigin="anonymous"></script>
{% block scripts %}
{% endblock %}
</body>
</html>

View File

@ -0,0 +1,38 @@
{% extends "base.j2" %}
{% block title %}
PaperParser: Главная
{% endblock title %}
{% block main %}
<div class="container">
<div id="cards-grid" class="row row-cols-1 row-cols-sm-2 row-cols-md-3 row-cols-lg-4">
{% for session in sessions %}
<div class="col mb-3" data-session="{{ session.name }}">
<div class="card">
<div class="card-body">
<h3 class="card-title">{{ session.name }}</h3>
</div>
<div class="card-footer">
<a href="/parse/{{ session.id }}" class="btn btn-outline-primary mt-2">
Использовать
</a>
<button
hx-delete="/api/sessions/{{ session.id }}"
hx-swap="delete"
hx-target='[data-session="{{ session.name }}"]'
hx-confirm="Вы уверены, что хотите удалить эту сессию?"
class="btn btn-outline-danger mt-2 mx-2"
>
Удалить
</button>
</div>
</div>
</div>
{% endfor %}
</div>
</div>
{% endblock main %}

View File

@ -0,0 +1,101 @@
{% extends "base.j2" %}
{% block title %}
PaperParser: Парсинг
{% endblock title %}
{% block main %}
<div class="container">
<!-- Actions -->
<section id="actions-section" class="mb-3">
<div class="separator">
<h2>Действия</h2>
<hr class="divider">
</div>
<section id="actions-parse-add-section">
<label for="" class="form-label">Парсинг пользователей и добавление в группу</label>
<div class="row row-cols-1 row-cols-sm-2">
<div class="col mb-3">
<form hx-post="/api/users/parse/{{session_id}}" hx-swap="innerHTML" hx-target="#cards-grid" hx-indicator="#loading-spinner" class="input-group" id="group-from-form">
<div class="form-floating">
<input type="text" class="form-control" name="group" id="group-from-input"
placeholder="Группа для парсинга">
<label for="group-from-input">Группа для парсинга</label>
</div>
<button type="submit" class="btn btn-outline-primary">Спарсить</button>
</form>
</div>
<div class="col mb-3">
<form hx-post="/api/tasks/session/{{session_id}}" hx-swap="none" hx-indicator="#loading-spinner" class="input-group" id="group-to-form">
<div class="form-floating">
<input type="text" name="url" class="form-control" id="group-to-input"
placeholder="Группа, в которую нужно добавить">
<label for="group-to-input">Группа для добавления</label>
</div>
<button type="submit" name="task" value="add" class="btn btn-outline-secondary">Добавить</button>
</form>
</div>
</div>
</section>
<section id="actions-message-section">
<form hx-post="/api/tasks/session/{{session_id}}" hx-swap="none" hx-indicator="#loading-spinner" id="message-form">
<label for="" class="form-label">Сообщение для пользователей</label>
<div class="row row-cols-1 row-cols-sm-2">
<div class="col">
<div class="mb-3">
<textarea class="form-control" name="message" rows="8" id="message-textarea"
placeholder="Напишите здесь своё сообщение"></textarea>
</div>
</div>
<div class="col">
<div class="mb-3">
<label for="formFile" class="form-label">Выберите изображение</label>
<input class="form-control" name="file" type="file" id="picture-file-input">
</div>
<div class="mb-3">
<button type="submit" name="task" value="message" class="btn btn-outline-primary" style="width: 100%;">
Отправить сообщение
</button>
</div>
</div>
</div>
</form>
</section>
</section>
<!-- Users -->
<section id="users-section" class="mb-3">
<div id="separator">
<div class="d-flex justify-content-between">
<div class="d-flex gap-2 align-items-center">
<h2 style="margin-bottom: 0;">Пользователи</h2>
<div class="spinner-container htmx-indicator" id="loading-spinner">
<div class="spinner-border" role="status">
<span class="visually-hidden">Loading...</span>
</div>
</div>
</div>
<div>
<button class="btn btn-outline-primary disabled" onClick="exportToCSV()">Экспорт</button>
<button
class="btn btn-outline-danger"
hx-delete="/api/users/{{session_id}}"
hx-swap="innerHTML"
hx-target="#cards-grid"
hx-confirm="Вы уверены, что хотите удалить пользователей для этой сессии?"
hx-indicator="#loading-spinner"
>
Удалить
</button>
</div>
</div>
<hr class="divider">
</div>
<div id="cards-grid" class="row row-cols-1 row-cols-sm-2 row-cols-md-3 row-cols-lg-4">
{{ users_template }}
</div>
</section>
</div>
{% endblock main %}

View File

@ -0,0 +1,16 @@
{% extends "base.j2" %}
{% block title %}
PaperParser: Заадчи
{% endblock title %}
{% block main %}
<div class="container">
<div class="separator">
<h2>Задачи</h2>
<hr class="divider">
</div>
<div id="cards-grid" class="row row-cols-1 row-cols-sm-2 row-cols-md-3 row-cols-lg-4">
{{ tasks_template }}
</div>
{% endblock main %}

18
app/celery.py Normal file
View File

@ -0,0 +1,18 @@
from flask import Flask
from celery import Celery
from celery.contrib.abortable import AbortableTask
from app.blueprints.api.tasks.handlers import *
def celery_init_app(app: Flask) -> Celery:
class FlaskTask(AbortableTask):
def __call__(self, *args: object, **kwargs: object) -> object:
with app.app_context():
return self.run(*args, **kwargs)
celery_app = Celery(app.name, task_cls=FlaskTask)
celery_app.config_from_object(app.config["CELERY"])
celery_app.set_default()
app.extensions["celery"] = celery_app
return celery_app

5
app/extensions.py Normal file
View File

@ -0,0 +1,5 @@
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
db = SQLAlchemy()
migrate = Migrate(db=db)

24
app/models/session.py Normal file
View File

@ -0,0 +1,24 @@
from typing import List
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.models.task import Task
from app.models.user import User
from app.extensions import db
class Session(db.Model):
id: Mapped[int] = mapped_column(primary_key=True)
name: Mapped[str] = mapped_column(unique=True)
authorized: Mapped[bool]
users: Mapped[List[User]] = relationship("User", back_populates="session")
tasks: Mapped[List[Task]] = relationship("Task", back_populates="session")
def has_depending_tasks(self):
tasks = Task.query \
.filter_by(session_id=self.id) \
.filter((Task.status == "RUNNING") | (Task.status == "RUNNING")) \
.all()
return len(tasks) > 0

20
app/models/task.py Normal file
View File

@ -0,0 +1,20 @@
from sqlalchemy import Integer
from sqlalchemy.schema import Column, ForeignKey
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.extensions import db
class Task(db.Model):
id: Mapped[int] = mapped_column(primary_key=True)
task_id: Mapped[str] = mapped_column(nullable=True)
name: Mapped[str] = mapped_column(unique=True)
session_id = Column(Integer, ForeignKey("session.id"))
session: Mapped['Session'] = relationship("Session", back_populates="tasks")
status: Mapped[str]
status_message: Mapped[str]
type: Mapped[str]
url: Mapped[str] = mapped_column(nullable=True)
message: Mapped[str] = mapped_column(nullable=True)
file: Mapped[str] = mapped_column(nullable=True)

19
app/models/user.py Normal file
View File

@ -0,0 +1,19 @@
from sqlalchemy import Integer
from sqlalchemy.schema import (
Column,
ForeignKey,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.extensions import db
class User(db.Model):
id: Mapped[int] = mapped_column(primary_key=True)
first_name: Mapped[str] = mapped_column(nullable=True)
last_name: Mapped[str] = mapped_column(nullable=True)
phone: Mapped[str] = mapped_column(nullable=True, unique=True)
username: Mapped[str] = mapped_column(nullable=True, unique=True)
session_id = Column(Integer, ForeignKey("session.id"))
session: Mapped["Session"] = relationship("Session", back_populates="users")

BIN
app/static/favicon.ico Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

15
app/utils/__init__.py Normal file
View File

@ -0,0 +1,15 @@
import asyncio
import functools
def to_sync_task(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
try:
loop = asyncio.get_event_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_forever()
return loop.run_until_complete(func(*args, **kwargs))
return wrapper

14
config.py Normal file
View File

@ -0,0 +1,14 @@
import os
basedir = os.path.abspath(os.path.dirname(__file__))
class Config:
SECRET_KEY = os.environ.get('SECRET_KEY')
SQLALCHEMY_DATABASE_URI = f"postgresql://{os.environ.get('POSTGRES_USER')}:{os.environ.get('POSTGRES_PASSWORD')}@postgres/{os.environ.get('POSTGRES_DB')}"
SQLALCHEMY_TRACK_MODIFICATIONS = False
CELERY = dict(
broker_url="redis://redis/0",
result_backend="redis://redis/0",
task_ignore_result=True,
)

29
docker-compose.yml Normal file
View File

@ -0,0 +1,29 @@
services:
redis:
image: redis:7
postgres:
env_file:
- .env
image: postgres:16
celery:
build:
context: .
env_file:
- .env
command: celery -A make_celery worker --loglevel INFO
depends_on:
- redis
web:
build:
context: .
env_file:
- .env
command: gunicorn --bind 0.0.0.0:5000 "app:create_app()"
ports:
- 5000:5000
depends_on:
- celery
- postgres

4
example.env Normal file
View File

@ -0,0 +1,4 @@
FLASK_APP=
POSTGRES_USER=
POSTGRES_PASSWORD=
POSTGRES_DB=

4
make_celery.py Normal file
View File

@ -0,0 +1,4 @@
from app import create_app
flask_app = create_app()
celery_app = flask_app.extensions['celery']

1
migrations/README Normal file
View File

@ -0,0 +1 @@
Single-database configuration for Flask.

50
migrations/alembic.ini Normal file
View File

@ -0,0 +1,50 @@
# A generic, single database configuration.
[alembic]
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic,flask_migrate
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[logger_flask_migrate]
level = INFO
handlers =
qualname = flask_migrate
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

113
migrations/env.py Normal file
View File

@ -0,0 +1,113 @@
import logging
from logging.config import fileConfig
from flask import current_app
from alembic import context
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
def get_engine():
try:
# this works with Flask-SQLAlchemy<3 and Alchemical
return current_app.extensions['migrate'].db.get_engine()
except (TypeError, AttributeError):
# this works with Flask-SQLAlchemy>=3
return current_app.extensions['migrate'].db.engine
def get_engine_url():
try:
return get_engine().url.render_as_string(hide_password=False).replace(
'%', '%%')
except AttributeError:
return str(get_engine().url).replace('%', '%%')
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
config.set_main_option('sqlalchemy.url', get_engine_url())
target_db = current_app.extensions['migrate'].db
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def get_metadata():
if hasattr(target_db, 'metadatas'):
return target_db.metadatas[None]
return target_db.metadata
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url, target_metadata=get_metadata(), literal_binds=True
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
conf_args = current_app.extensions['migrate'].configure_args
if conf_args.get("process_revision_directives") is None:
conf_args["process_revision_directives"] = process_revision_directives
connectable = get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=get_metadata(),
**conf_args
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
migrations/script.py.mako Normal file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
branch_labels = ${repr(branch_labels)}
depends_on = ${repr(depends_on)}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,47 @@
"""Innitial commit
Revision ID: 31cd88d78cb4
Revises:
Create Date: 2024-01-21 14:31:12.188455
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '31cd88d78cb4'
down_revision = None
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('session',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('name', sa.String(), nullable=False),
sa.Column('authorized', sa.Boolean(), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('user',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('first_name', sa.String(), nullable=True),
sa.Column('last_name', sa.String(), nullable=True),
sa.Column('phone', sa.String(), nullable=True),
sa.Column('username', sa.String(), nullable=True),
sa.Column('session_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['session_id'], ['session.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('phone'),
sa.UniqueConstraint('username')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('user')
op.drop_table('session')
# ### end Alembic commands ###

View File

@ -0,0 +1,42 @@
"""Deleted json filed
Revision ID: af22ea98e0da
Revises: 31cd88d78cb4
Create Date: 2024-01-22 02:14:59.504400
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'af22ea98e0da'
down_revision = '31cd88d78cb4'
branch_labels = None
depends_on = None
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('task',
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('task_id', sa.String(), nullable=True),
sa.Column('name', sa.String(), nullable=False),
sa.Column('session_id', sa.Integer(), nullable=True),
sa.Column('status', sa.String(), nullable=False),
sa.Column('status_message', sa.String(), nullable=False),
sa.Column('type', sa.String(), nullable=False),
sa.Column('url', sa.String(), nullable=True),
sa.Column('message', sa.String(), nullable=True),
sa.Column('file', sa.String(), nullable=True),
sa.ForeignKeyConstraint(['session_id'], ['session.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('task')
# ### end Alembic commands ###

0
paper/__init__.py Normal file
View File

103
paper/client.py Normal file
View File

@ -0,0 +1,103 @@
from loguru import logger
from opentele.tl import TelegramClient
from opentele.api import API
from telethon.tl.functions.channels import InviteToChannelRequest, JoinChannelRequest
from telethon.tl.functions.messages import AddChatUserRequest
# Types
from telethon.hints import (Entity, EntityLike, MessageLike)
from telethon.types import Channel, Chat, User
from telethon.sessions.abstract import Session
from paper.errors import *
from telethon.errors.rpcerrorlist import (
ChatAdminRequiredError,
UserPrivacyRestrictedError,
UserNotMutualContactError,
ChannelPrivateError,
ChatWriteForbiddenError,
PeerFloodError,
FloodWaitError,
UsersTooMuchError,
UserChannelsTooMuchError,
UserIsBlockedError,
YouBlockedUserError
)
class PaperClient(TelegramClient):
def __init__(self, session: str | Session) -> None:
api = API.TelegramAndroid.Generate("paper")
super().__init__(session, api)
async def invite_self(self, group: Entity | EntityLike):
group = await self.__cast_to_entity(group)
if isinstance(group, Channel):
await self(JoinChannelRequest(group))
logger.info(f"Added self to a channel {group.title}")
@logger.catch(reraise=True)
async def invite_user(self, user: Entity | EntityLike, group: Entity | EntityLike):
user, group = await self.__cast_to_entity(user), await self.__cast_to_entity(group)
try:
if isinstance(group, Channel) and isinstance(user, User):
await self(InviteToChannelRequest(group, [user])) # type: ignore
logger.info(f"User {user} was added to channel {group.title}")
elif isinstance(group, Chat) and isinstance(user, User):
await self(AddChatUserRequest(group, user, 30)) # type: ignore
logger.info(f"User {user} was added to chat {group.title}")
else:
logger.warning(f"Can't determine group type for {group}. Skipping...")
except (UserPrivacyRestrictedError, UserNotMutualContactError) as e:
raise UserPrivacyException(e)
except (ChatWriteForbiddenError, ChannelPrivateError, ChatAdminRequiredError) as e:
raise AdminPrivilegesExceptions(e)
except (PeerFloodError, FloodWaitError) as e:
raise FloodException(e)
except (UserChannelsTooMuchError, UsersTooMuchError, UserIsBlockedError, YouBlockedUserError) as e:
raise IgnoreException(e)
@logger.catch(reraise=True)
async def send_message(self, entity: Entity | EntityLike, message: str, file: str | None):
entity = await self.__cast_to_entity(entity)
try:
return await super().send_message(entity, message=message, file=file) #type: ignore
except (UserPrivacyRestrictedError, UserNotMutualContactError) as e:
raise UserPrivacyException(e)
except (ChatWriteForbiddenError, ChannelPrivateError, ChatAdminRequiredError) as e:
raise AdminPrivilegesExceptions(e)
except (PeerFloodError, FloodWaitError) as e:
raise FloodException(e)
except (UserChannelsTooMuchError, UsersTooMuchError, UserIsBlockedError, YouBlockedUserError) as e:
raise IgnoreException(e)
@logger.catch(reraise=True)
async def get_participants(self, group: Entity | EntityLike, *args, **kwargs):
group = await self.__cast_to_entity(group)
return await super().get_participants(group, *args, **kwargs)
async def __cast_to_entity(self, entity: Entity | EntityLike) -> Entity:
if not isinstance(entity, Entity):
if hasattr(entity, "username"):
entity = await self.get_entity(entity.username)
else:
entity = await self.get_entity(entity)
return entity # type: ignore

40
paper/errors/__init__.py Normal file
View File

@ -0,0 +1,40 @@
class UserPrivacyException(Exception):
"""Occurs when user's privacy setting doesn't allow certain actions (e.g., inviting to a channel, sending a message)"""
def __init__(self, error: Exception, *args: object) -> None:
self.error = error
self.message = f"Can't do action due to user's privacy settings"
super().__init__(self.message, *args)
class AdminPrivilegesExceptions(Exception):
"""Occurs when administrator privileges are required to perform a certain action (e.g., channel invitations, user parsing)"""
def __init__(self, error: Exception, *args: object) -> None:
self.error = error
self.message = f"Administrator privileges are required to perform the action."
super().__init__(self.message, *args)
class FloodException(Exception):
def __init__(self, error, *args: object):
self.error = error
self.seconds = self.error.seconds if hasattr(self.error, "seconds") else -1 #type: ignore
super().__init__(*args)
class IgnoreException(Exception):
def __init__(self, error, *args):
self.error = error
super().__init__(*args)
class TooMuchException(Exception):
def __init__(self, error, *args):
self.error = error
super().__init__(*args)
class NeedPasswordException(Exception):
def __init__(self, *args):
self.message = "Need password"
super().__init__(self.message, *args)

13
paper/models.py Normal file
View File

@ -0,0 +1,13 @@
from dataclasses import dataclass
from paper.utils.classes import DataclassBase
dataclass(init=False)
class Message(DataclassBase):
text: str = ''
images: None | list[str] = None
force_document: bool = False
def exists(self):
return self.text or self.images

92
paper/parser.py Normal file
View File

@ -0,0 +1,92 @@
import asyncio
import os
from loguru import logger
from paper.client import PaperClient
from paper.errors import IgnoreException, NeedPasswordException, UserPrivacyException
from paper.models import Message
class PaperParser:
def __init__(self, session: str) -> None:
if not session:
raise ValueError("Session name can't be None")
self.client = PaperClient(session)
self.users_to_delete = []
async def invite_users(self, users, group, task = None):
await self.client.invite_self(group)
group_participants = await self.client.get_participants(group)
if group_participants:
participants_usernames = [participant.username for participant in group_participants if participant.username]
users = filter(lambda user: user.username not in participants_usernames, users)
for user in users:
try:
if task.is_aborted():
return self.users_to_delete
await self.client.invite_user(user, group)
except (UserPrivacyException, IgnoreException) as e:
self.users_to_delete.append(user)
logger.exception(e)
logger.warning("Exception occurred. Skipping user...")
except Exception as e:
logger.exception(e)
finally:
if not task.is_aborted():
await asyncio.sleep(50) # FIXME: Change to config value
async def send_messages(self, users, message: str, file: str | None = None, task = None):
# TODO: Filter users, that already get this message
# for dialog in await self.client.get_dialogs():
# if dialog.is_user:
# messages = tuple(filter(lambda chat_message: message.text == chat_message.text, await self.client.get_messages(dialog)))
# print(messages)
for user in users:
try:
if task.is_aborted():
return self.users_to_delete
await self.client.send_message(user, message, file)
except (UserPrivacyException, IgnoreException) as e:
self.users_to_delete.append(user)
logger.exception(e)
logger.warning("Exception occurred. Skipping user...")
finally:
if not task.is_aborted():
await asyncio.sleep(50) # FIXME: Change to config value
async def get_participants(self, group):
await self.client.invite_self(group)
return await self.client.get_participants(group)
async def sign_in(self, phone: str, password: str | None = None, code: str | None = None, phone_hash: str | None = None, **kwargs):
if not await self.client.is_user_authorized():
try:
return await self.client.sign_in(phone=phone, code=code, phone_code_hash=phone_hash) #type: ignore
except:
if not password:
raise NeedPasswordException()
return await self.client.sign_in(password=password)
async def __aenter__(self, *args, **kwargs):
await self.client.connect()
return self
async def __aexit__(self, *args, **kwargs):
await self.client.disconnect()

20
paper/utils/classes.py Normal file
View File

@ -0,0 +1,20 @@
import dataclasses
@dataclasses.dataclass(init=False)
class DataclassBase:
"""
It just works
"""
def __init__(self, **kwargs):
names = set([f.name for f in dataclasses.fields(self)])
for k, v in kwargs.items():
if k in names:
setattr(self, k, v)
def dict(self):
return {
k: v.dict() if isinstance(v, DataclassBase) else v
for k, v in dataclasses.asdict(self).items()
}

43
requirements.txt Normal file
View File

@ -0,0 +1,43 @@
alembic==1.13.1
amqp==5.2.0
asgiref==3.7.2
async-timeout==4.0.3
billiard==4.2.0
blinker==1.7.0
celery==5.3.6
click==8.1.7
click-didyoumean==0.3.0
click-plugins==1.1.1
click-repl==0.3.0
Flask==3.0.1
Flask-Migrate==4.0.5
Flask-SQLAlchemy==3.1.1
greenlet==3.0.3
gunicorn==21.2.0
itsdangerous==2.1.2
Jinja2==3.1.3
kombu==5.3.5
loguru==0.7.2
Mako==1.3.0
MarkupSafe==2.1.4
opentele==1.15.1
packaging==23.2
prompt-toolkit==3.0.43
psycopg2-binary==2.9.9
pyaes==1.6.1
pyasn1==0.5.1
PyQt5==5.15.10
PyQt5-Qt5==5.15.2
PyQt5-sip==12.13.0
python-dateutil==2.8.2
redis==5.0.1
rsa==4.9
six==1.16.0
SQLAlchemy==2.0.25
Telethon==1.33.1
TgCrypto==1.2.5
typing_extensions==4.9.0
tzdata==2023.4
vine==5.1.0
wcwidth==0.2.13
Werkzeug==3.0.1