From d21542a9ad05be5ee370bdbe9e1e36e5aa32a72c Mon Sep 17 00:00:00 2001 From: Anatoly Bogomolov Date: Wed, 31 Jan 2024 19:37:01 +1000 Subject: [PATCH] Initial commit --- .gitignore | 4 + .vscode/launch.json | 21 ++++ Dockerfile | 11 ++ app/__init__.py | 20 +++ app/blueprints/__init__.py | 7 ++ app/blueprints/api/routes.py | 11 ++ app/blueprints/api/sessions/routes.py | 81 ++++++++++++ app/blueprints/api/tasks/handlers.py | 27 ++++ app/blueprints/api/tasks/routes.py | 83 +++++++++++++ app/blueprints/api/tasks/tasks.py | 47 +++++++ .../api/tasks/templates/tasks_cards.j2 | 34 ++++++ app/blueprints/api/users/routes.py | 58 +++++++++ .../api/users/templates/user_cards.j2 | 18 +++ app/blueprints/frontend/routes.py | 23 ++++ app/blueprints/frontend/static/add.js | 115 ++++++++++++++++++ app/blueprints/frontend/templates/add.j2 | 105 ++++++++++++++++ app/blueprints/frontend/templates/base.j2 | 64 ++++++++++ app/blueprints/frontend/templates/index.j2 | 38 ++++++ app/blueprints/frontend/templates/parse.j2 | 101 +++++++++++++++ app/blueprints/frontend/templates/tasks.j2 | 16 +++ app/celery.py | 18 +++ app/extensions.py | 5 + app/models/session.py | 24 ++++ app/models/task.py | 20 +++ app/models/user.py | 19 +++ app/static/favicon.ico | Bin 0 -> 4286 bytes app/utils/__init__.py | 15 +++ config.py | 14 +++ docker-compose.yml | 29 +++++ example.env | 4 + make_celery.py | 4 + migrations/README | 1 + migrations/alembic.ini | 50 ++++++++ migrations/env.py | 113 +++++++++++++++++ migrations/script.py.mako | 24 ++++ .../versions/31cd88d78cb4_innitial_commit.py | 47 +++++++ .../af22ea98e0da_deleted_json_filed.py | 42 +++++++ paper/__init__.py | 0 paper/client.py | 103 ++++++++++++++++ paper/errors/__init__.py | 40 ++++++ paper/models.py | 13 ++ paper/parser.py | 92 ++++++++++++++ paper/utils/classes.py | 20 +++ requirements.txt | 43 +++++++ 44 files changed, 1624 insertions(+) create mode 100644 .vscode/launch.json create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/blueprints/__init__.py create mode 100644 app/blueprints/api/routes.py create mode 100644 app/blueprints/api/sessions/routes.py create mode 100644 app/blueprints/api/tasks/handlers.py create mode 100644 app/blueprints/api/tasks/routes.py create mode 100644 app/blueprints/api/tasks/tasks.py create mode 100644 app/blueprints/api/tasks/templates/tasks_cards.j2 create mode 100644 app/blueprints/api/users/routes.py create mode 100644 app/blueprints/api/users/templates/user_cards.j2 create mode 100644 app/blueprints/frontend/routes.py create mode 100644 app/blueprints/frontend/static/add.js create mode 100644 app/blueprints/frontend/templates/add.j2 create mode 100644 app/blueprints/frontend/templates/base.j2 create mode 100644 app/blueprints/frontend/templates/index.j2 create mode 100644 app/blueprints/frontend/templates/parse.j2 create mode 100644 app/blueprints/frontend/templates/tasks.j2 create mode 100644 app/celery.py create mode 100644 app/extensions.py create mode 100644 app/models/session.py create mode 100644 app/models/task.py create mode 100644 app/models/user.py create mode 100644 app/static/favicon.ico create mode 100644 app/utils/__init__.py create mode 100644 config.py create mode 100644 docker-compose.yml create mode 100644 example.env create mode 100644 make_celery.py create mode 100644 migrations/README create mode 100644 migrations/alembic.ini create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/31cd88d78cb4_innitial_commit.py create mode 100644 migrations/versions/af22ea98e0da_deleted_json_filed.py create mode 100644 paper/__init__.py create mode 100644 paper/client.py create mode 100644 paper/errors/__init__.py create mode 100644 paper/models.py create mode 100644 paper/parser.py create mode 100644 paper/utils/classes.py create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 5d381cc..92ab989 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,7 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +*.db +*.session +.vscode/settings.json +*session_journal diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..a125546 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + "configurations": [ + { + "name": "Python: Flask", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "app", + "FLASK_DEBUG": "1" + }, + "args": [ + "run", + "--no-debugger", + "--no-reload", + ], + "jinja": true, + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ba897f4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11 + +COPY app/ /app/ +COPY migrations/ /migrations/ +COPY paper /paper/ + +COPY config.py config.py +COPY make_celery.py make_celery.py +COPY requirements.txt requirements.txt + +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e6fade9 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,20 @@ +from flask import Flask + +from config import Config +from app.blueprints import blueprints +from app.extensions import db, migrate +from app.celery import celery_init_app + +def create_app(config_class=Config): + app = Flask(__name__) + app.config.from_object(config_class) + + db.init_app(app) + migrate.init_app(app) + + for blueprint in blueprints: + app.register_blueprint(blueprint) + + celery_init_app(app) + + return app \ No newline at end of file diff --git a/app/blueprints/__init__.py b/app/blueprints/__init__.py new file mode 100644 index 0000000..c7775ff --- /dev/null +++ b/app/blueprints/__init__.py @@ -0,0 +1,7 @@ +from .api.routes import api +from .frontend.routes import frontend + +blueprints = ( + api, + frontend, +) \ No newline at end of file diff --git a/app/blueprints/api/routes.py b/app/blueprints/api/routes.py new file mode 100644 index 0000000..0cd1db4 --- /dev/null +++ b/app/blueprints/api/routes.py @@ -0,0 +1,11 @@ +from flask import Blueprint + +from .tasks.routes import tasks +from .sessions.routes import sessions +from .users.routes import users + +api = Blueprint("api", __name__, url_prefix="/api") + +api.register_blueprint(users) +api.register_blueprint(tasks) +api.register_blueprint(sessions) \ No newline at end of file diff --git a/app/blueprints/api/sessions/routes.py b/app/blueprints/api/sessions/routes.py new file mode 100644 index 0000000..ce749e4 --- /dev/null +++ b/app/blueprints/api/sessions/routes.py @@ -0,0 +1,81 @@ +from flask import Blueprint, Response, jsonify, request + +from app.extensions import db +from app.models.session import Session +from app.models.task import Task +from paper.errors import NeedPasswordException +from paper.parser import PaperParser + +sessions = Blueprint("sessions", __name__, url_prefix="/sessions") + +@sessions.route("/", methods=["POST"]) +async def create_session(**kwargs): + data = kwargs or request.json or request.form + data["name"] = data["name"].replace("/", "").replace("\\", "") + + session = Session.query.filter_by(name=data.get("name")).first() + + if not session: + session = Session( + name=data.get("name"), + authorized=False + ) + + db.session.add(session) + + response = {"status": "ok"} + + async with PaperParser(data.get("name")) as parser: + try: + result = await parser.sign_in(**data) + + if hasattr(result, "phone_code_hash"): + response = { + "status": "error", + "message": "Need secret code from telegram", + "action": "code", + "phone_hash": result.phone_code_hash, + } + else: + session.authorized = True + + except NeedPasswordException: + response = { + "status": "error", + "message": "Need 2FA Password", + "action": "password", + } + + except Exception as e: + response = { + "status": "exception", + "message": str(e), + } + + finally: + db.session.commit() + + return jsonify(response) + +@sessions.route("/", methods=["GET"]) +def get_sessions(**kwargs): + return Session.query.filter_by(authorized=True).all() + +@sessions.route("/", methods=["DELETE"]) +async def remove_session(id: int): + session: Session = Session.query.get_or_404(id) + + if session.has_depending_tasks(): + return 'Есть зависимые задачи', 409 + + async with PaperParser(session.name) as parser: + try: + await parser.client.log_out() + + except Exception as e: + return Response(e, 500) + + db.session.delete(session) + db.session.commit() + + return Response('ok', status=200) \ No newline at end of file diff --git a/app/blueprints/api/tasks/handlers.py b/app/blueprints/api/tasks/handlers.py new file mode 100644 index 0000000..d5d72fc --- /dev/null +++ b/app/blueprints/api/tasks/handlers.py @@ -0,0 +1,27 @@ +from app.extensions import db +from app.models.task import Task + +def success_state(task_id): + task = Task.query.filter_by(task_id=task_id).one() + + task.status = "SUCCESS" + task.status_message = "Задача выполнена успешно" + + db.session.commit() + +def failure_state(task_id, exception): + task = Task.query.filter_by(task_id=task_id).one() + + task.status = "FAILURE" + task.status_message = f"Произошла ошибка: {str(exception)}" + + db.session.commit() + +def run_state(task_id, task_record_id): + task = Task.query.get(task_record_id) + + task.task_id = task_id + task.status = "RUNNING" + task.status_message = "Задача запущена" + + db.session.commit() \ No newline at end of file diff --git a/app/blueprints/api/tasks/routes.py b/app/blueprints/api/tasks/routes.py new file mode 100644 index 0000000..3f8a76a --- /dev/null +++ b/app/blueprints/api/tasks/routes.py @@ -0,0 +1,83 @@ +import uuid +from flask import Blueprint, render_template, request + +from app.extensions import db +from app.models.task import Task +from app.models.session import Session + +from .tasks import send_messages_task, add_to_group_task + +tasks = Blueprint("tasks", __name__, url_prefix="/tasks", template_folder="templates") + +@tasks.route("/", methods=["GET"]) +def get_tasks(): + tasks = Task.query.all() + + return render_template("tasks_cards.j2", tasks=tasks) + +@tasks.route("/", methods=["GET"]) +def get_task(task_id: int): + ... + +@tasks.route("/session/", methods=["POST"]) +def create_task(session_id: int): + info = request.form + session = Session.query.get_or_404(session_id) + task = Task( + name = str(uuid.uuid4()), + session = session, + status = "CREATED", + status_message = "Задача создана", + type = info.get('task'), + url = info.get('url'), + message = info.get('message'), + file = None + ) + + db.session.add(task) + db.session.commit() + + start_task(task.id) + + return 'Created', 200 + +@tasks.route("/", methods=["DELETE"]) +def delete_task(task_id: int): + task: Task = Task.query.get_or_404(task_id) + stop_task(task.id) + db.session.delete(task) + db.session.commit() + + return 'Deleted', 200 + +@tasks.route("//stop", methods=["PUT"]) +def stop_task(task_id: int): + task = Task.query.get_or_404(task_id) + fn = None + + match task.type: + case 'message': + fn = send_messages_task + + case 'add': + fn = add_to_group_task + + if task.task_id: + task_fn = fn.AsyncResult(task.task_id) + task_fn.abort() + + return 'Stopped', 200 + + +@tasks.route("//start", methods=["PUT"]) +def start_task(task_id: int): + task: Task = Task.query.get_or_404(task_id) + + match task.type: + case 'message': + send_messages_task.delay(task_id=task_id) + + case 'add': + add_to_group_task.delay(task_id=task_id) + + return 'Started', 200 \ No newline at end of file diff --git a/app/blueprints/api/tasks/tasks.py b/app/blueprints/api/tasks/tasks.py new file mode 100644 index 0000000..c4a3e5c --- /dev/null +++ b/app/blueprints/api/tasks/tasks.py @@ -0,0 +1,47 @@ +import time + +from celery import shared_task +import asyncio + +from paper.parser import PaperParser +from app.models.task import Task +from app.extensions import db +from .handlers import run_state, failure_state, success_state + +async def add_to_group(session, task, task_self): + async with PaperParser(session.name) as parser: + await parser.invite_users(session.users, task.url, task_self) + +async def sending_message(session, task, task_self): + async with PaperParser(session.name) as parser: + await parser.send_messages(session.users, task.message, task.file, task_self) + +@shared_task(bind=True) +def add_to_group_task(self, task_id): + try: + task: Task = Task.query.get(task_id) + run_state(self.request.id, task_id) + + session = task.session + + time.sleep(10) + + asyncio.run( + add_to_group(session, task, self) + ) + + except Exception as e: + failure_state(self.request.id, e) + raise e + + else: + success_state(self.request.id) + + + +@shared_task(bind=True) +def send_messages_task(self, task_id): + task: Task = Task.query.get(task_id) + + session = task.session + users = session.users \ No newline at end of file diff --git a/app/blueprints/api/tasks/templates/tasks_cards.j2 b/app/blueprints/api/tasks/templates/tasks_cards.j2 new file mode 100644 index 0000000..d102e0e --- /dev/null +++ b/app/blueprints/api/tasks/templates/tasks_cards.j2 @@ -0,0 +1,34 @@ +{% for task in tasks %} +
+
+
+

{{ task.name }}

+
    +
  • Статус: {{ task.status }}
    • +
  • Сообщение: {{ task.status_message }}
    • +
  • Действие: {{ task.type }}
    • +
  • Сессия: {{ task.session.name }}
    • +
+
+
+ + + +
+
+
+{% else %} + Здесь ничего нет. +{% endfor %} \ No newline at end of file diff --git a/app/blueprints/api/users/routes.py b/app/blueprints/api/users/routes.py new file mode 100644 index 0000000..3f85d93 --- /dev/null +++ b/app/blueprints/api/users/routes.py @@ -0,0 +1,58 @@ +from flask import Blueprint, render_template, request + +from app.models.session import Session +from app.models.user import User +from app.extensions import db +from paper.parser import PaperParser + + +users = Blueprint("users", __name__, url_prefix="/users", template_folder="templates") + + +@users.route("/parse/", methods=["POST"]) +async def parse_users(session_id: int): + session = Session.query.get_or_404(session_id) + + data = request.form + + async with PaperParser(session.name) as parser: + users = await parser.get_participants(data.get("group")) + + for user in users: + if not user.username: + continue + + exists = db.session.query(User.id).filter_by(username=user.username).first() + if not exists: + db.session.add( + User( + first_name=user.first_name, + last_name=user.last_name, + username=user.username, + phone=user.phone, + session=session, + ) + ) + + db.session.commit() + + return render_template("user_cards.j2", users=session.users) + + +@users.route("/", methods=["GET"]) +def get_users(session_id: int): + users = Session.query.get_or_404(session_id).users + + return render_template("user_cards.j2", users=users) + +@users.route("/", methods=["DELETE"]) +def delete_users(session_id): + session: Session = Session.query.get_or_404(session_id) + + if session.has_depending_tasks(): + return 'Есть зависимые задачи', 409 + + User.query.filter_by(session=session).delete() + db.session.commit() + + return render_template("user_cards.j2", users=[]) \ No newline at end of file diff --git a/app/blueprints/api/users/templates/user_cards.j2 b/app/blueprints/api/users/templates/user_cards.j2 new file mode 100644 index 0000000..56a7c8e --- /dev/null +++ b/app/blueprints/api/users/templates/user_cards.j2 @@ -0,0 +1,18 @@ +{% for user in users %} +
+
+
+

{{ user.first_name }}

+
+ +
+
+{% else %} + Здесь ничего нет. +{% endfor %} \ No newline at end of file diff --git a/app/blueprints/frontend/routes.py b/app/blueprints/frontend/routes.py new file mode 100644 index 0000000..b22e35b --- /dev/null +++ b/app/blueprints/frontend/routes.py @@ -0,0 +1,23 @@ +from flask import Blueprint, render_template +from app.blueprints.api.sessions.routes import get_sessions +from app.blueprints.api.users.routes import get_users +from app.blueprints.api.tasks.routes import get_tasks + +frontend = Blueprint("frontend", __name__, url_prefix="/", template_folder="templates", static_folder="static", static_url_path="/static/frontend") + +@frontend.route("/") +def index(): + sessions = get_sessions() + return render_template("index.j2", sessions=sessions) + +@frontend.route("/add") +def add(): + return render_template("add.j2") + +@frontend.route("/tasks") +def tasks(): + return render_template("tasks.j2", tasks_template=get_tasks()) + +@frontend.route("/parse/") +def parse(id: int): + return render_template("parse.j2", session_id=id, users_template=get_users(id)) \ No newline at end of file diff --git a/app/blueprints/frontend/static/add.js b/app/blueprints/frontend/static/add.js new file mode 100644 index 0000000..e975b83 --- /dev/null +++ b/app/blueprints/frontend/static/add.js @@ -0,0 +1,115 @@ +const API_URL = window.location.origin; + +const nameInput = document.querySelector("#input-name"); +const phoneInput = document.querySelector("#input-phone"); +const addSessionForm = document.querySelector("#add-session-form"); + +const openFileButton = document.querySelector("#open-file-btn"); + +const secretCodeModal = new bootstrap.Modal('#secretCodeModal'); +const secretCodeForm = document.querySelector("#secret-code-form"); +const secretCodeInput = document.querySelector("#input-secret-code"); + +const passwordModal = new bootstrap.Modal('#passwordModal'); +const passwordForm = document.querySelector("#password-form"); +const passwordInput = document.querySelector("#input-password"); + +let phoneHash = ""; + +const handleCreation = async (e) => { + e.preventDefault(); + + if (phoneInput.value == "" || nameInput.value == "") { + alert("Пожалуйста, заполните все поля."); + return; + } + + const data = getAuthData(); + + res = await fetch(`${API_URL}/api/sessions/`, { + method: "POST", + body: JSON.stringify(data), + headers: { + "Content-Type": "application/json", + } + }); + + result = await res.json(); + + switch(result.status) { + case "ok": + window.location = "/"; + break; + + case "error": + handleError(result); + break; + + case "exception": + alert(`Ошибка: ${result.message}.`); + break; + + default: + alert(`Неизвестная ошибка. Проверьте логи`); + } +} + +const getAuthData = () => { + const inputFields = [passwordInput, secretCodeInput, nameInput, phoneInput] + data = {} + inputFields.forEach((el) => { + if (el.value !== "") { + data[el.attributes.name.value] = el.value.replace(/ /g, ''); + } + }) + + if (phoneHash !== "") + data["phone_hash"] = phoneHash; + + return data; +} + +const handleRequest = async (fn) => { +} + +const handleError = (result) => { + switch(result.action) { + case "code": + phoneHash = result.phone_hash; + secretCodeModal.show(); + break; + case "password": + passwordModal.show(); + break; + } +} + +addSessionForm.addEventListener("submit", handleCreation); +secretCodeForm.addEventListener("submit", handleCreation); +passwordForm.addEventListener("submit", handleCreation); + + +// openFileButton.addEventListener("click", handleOpening); + +function handleOpening() { + let input = document.createElement('input'); + input.type = 'file'; + input.accept = ".zip"; + input.onchange = async () => { + // you can use this method to get file and perform respective operations + let archive = Array.from(input.files)[0]; + let formData = new FormData(); + + formData.append("archive", archive); + response = await fetch(`${API_URL}/api/save_session_file.php`, { method: "POST", body: formData }); + + if (response.status == "201") { + window.location = "/" + } else { + const error_text = await response.text(); + const error_status = await response.status; + alert(`${error_status}: ${error_text}`); + } + }; + input.click(); +} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/add.j2 b/app/blueprints/frontend/templates/add.j2 new file mode 100644 index 0000000..e407810 --- /dev/null +++ b/app/blueprints/frontend/templates/add.j2 @@ -0,0 +1,105 @@ +{% extends "base.j2" %} + +{% block title %} +PaperParser: Вход в аккаунт +{% endblock title %} + +{% block main %} +
+ + + + + + +
+ + + +
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+ +
+
+
+{% endblock main %} + +{% block scripts %} + +{% endblock scripts %} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/base.j2 b/app/blueprints/frontend/templates/base.j2 new file mode 100644 index 0000000..c49d152 --- /dev/null +++ b/app/blueprints/frontend/templates/base.j2 @@ -0,0 +1,64 @@ + + + + + {% block title %}{% endblock %} + + + + + + + + + + + +
+ + +
+
+ {% block main %} + + {% endblock %} +
+ + + + + + + + + + + + + {% block scripts %} + {% endblock %} + + \ No newline at end of file diff --git a/app/blueprints/frontend/templates/index.j2 b/app/blueprints/frontend/templates/index.j2 new file mode 100644 index 0000000..5eb8117 --- /dev/null +++ b/app/blueprints/frontend/templates/index.j2 @@ -0,0 +1,38 @@ +{% extends "base.j2" %} + +{% block title %} +PaperParser: Главная +{% endblock title %} + +{% block main %} +
+
+ + {% for session in sessions %} +
+
+
+

{{ session.name }}

+
+
+ + Использовать + + + +
+
+
+ {% endfor %} + +
+
+{% endblock main %} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/parse.j2 b/app/blueprints/frontend/templates/parse.j2 new file mode 100644 index 0000000..bbdc5d5 --- /dev/null +++ b/app/blueprints/frontend/templates/parse.j2 @@ -0,0 +1,101 @@ +{% extends "base.j2" %} + +{% block title %} +PaperParser: Парсинг +{% endblock title %} + +{% block main %} +
+ +
+
+

Действия

+
+
+
+ +
+
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+ +
+
+ +
+
+
+ +
+
+
+
+ + +
+
+ +
+
+
+
+
+
+ + +
+
+
+
+

Пользователи

+ +
+
+ + +
+ +
+
+
+
+ {{ users_template }} +
+
+
+{% endblock main %} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/tasks.j2 b/app/blueprints/frontend/templates/tasks.j2 new file mode 100644 index 0000000..27f6548 --- /dev/null +++ b/app/blueprints/frontend/templates/tasks.j2 @@ -0,0 +1,16 @@ +{% extends "base.j2" %} + +{% block title %} + PaperParser: Заадчи +{% endblock title %} + +{% block main %} +
+
+

Задачи

+
+
+
+ {{ tasks_template }} +
+{% endblock main %} \ No newline at end of file diff --git a/app/celery.py b/app/celery.py new file mode 100644 index 0000000..c5cdd85 --- /dev/null +++ b/app/celery.py @@ -0,0 +1,18 @@ +from flask import Flask +from celery import Celery +from celery.contrib.abortable import AbortableTask +from app.blueprints.api.tasks.handlers import * + +def celery_init_app(app: Flask) -> Celery: + class FlaskTask(AbortableTask): + def __call__(self, *args: object, **kwargs: object) -> object: + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name, task_cls=FlaskTask) + + celery_app.config_from_object(app.config["CELERY"]) + celery_app.set_default() + app.extensions["celery"] = celery_app + return celery_app + \ No newline at end of file diff --git a/app/extensions.py b/app/extensions.py new file mode 100644 index 0000000..8f3dacb --- /dev/null +++ b/app/extensions.py @@ -0,0 +1,5 @@ +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate + +db = SQLAlchemy() +migrate = Migrate(db=db) \ No newline at end of file diff --git a/app/models/session.py b/app/models/session.py new file mode 100644 index 0000000..900c8e7 --- /dev/null +++ b/app/models/session.py @@ -0,0 +1,24 @@ +from typing import List +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.task import Task +from app.models.user import User +from app.extensions import db + +class Session(db.Model): + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(unique=True) + + authorized: Mapped[bool] + users: Mapped[List[User]] = relationship("User", back_populates="session") + + tasks: Mapped[List[Task]] = relationship("Task", back_populates="session") + + def has_depending_tasks(self): + tasks = Task.query \ + .filter_by(session_id=self.id) \ + .filter((Task.status == "RUNNING") | (Task.status == "RUNNING")) \ + .all() + + return len(tasks) > 0 + \ No newline at end of file diff --git a/app/models/task.py b/app/models/task.py new file mode 100644 index 0000000..22676d3 --- /dev/null +++ b/app/models/task.py @@ -0,0 +1,20 @@ +from sqlalchemy import Integer +from sqlalchemy.schema import Column, ForeignKey +from sqlalchemy.orm import Mapped, mapped_column, relationship +from app.extensions import db + +class Task(db.Model): + id: Mapped[int] = mapped_column(primary_key=True) + task_id: Mapped[str] = mapped_column(nullable=True) + + name: Mapped[str] = mapped_column(unique=True) + + session_id = Column(Integer, ForeignKey("session.id")) + session: Mapped['Session'] = relationship("Session", back_populates="tasks") + status: Mapped[str] + status_message: Mapped[str] + type: Mapped[str] + + url: Mapped[str] = mapped_column(nullable=True) + message: Mapped[str] = mapped_column(nullable=True) + file: Mapped[str] = mapped_column(nullable=True) \ No newline at end of file diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 0000000..510b00b --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,19 @@ +from sqlalchemy import Integer +from sqlalchemy.schema import ( + Column, + ForeignKey, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.extensions import db + +class User(db.Model): + id: Mapped[int] = mapped_column(primary_key=True) + first_name: Mapped[str] = mapped_column(nullable=True) + last_name: Mapped[str] = mapped_column(nullable=True) + + phone: Mapped[str] = mapped_column(nullable=True, unique=True) + username: Mapped[str] = mapped_column(nullable=True, unique=True) + + session_id = Column(Integer, ForeignKey("session.id")) + session: Mapped["Session"] = relationship("Session", back_populates="users") \ No newline at end of file diff --git a/app/static/favicon.ico b/app/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..d880527becf832e5252b19a10374446f42769aab GIT binary patch literal 4286 zcmeHL`A=L`6n_2({$x^$m0&__3qygSKRAGtYHCXr#UK_a#V(;q5nK|bCe>Ie)KX1~ zOVXr7G_j%7P-uzRmIk2+3YA5ekz%nh3((n_eLBnMJ&!(|4JgAT{l!PlocHd#_k8F3 z&Y61wj(d###Kds)$u&OCaZhj@_Y5;O#APz0asFcOo)#_G!T`tN|0^s7yk0LlJ3G;F z{W`oJPgpV5vZ$-83tLlDu{9?Ty-Mv`Lalo3@bECQwr#`4=XXK!LkCI*XE1Jdt~yRQ z3nnKgk@NCP*u=1Rwe}(VcLjF#$?>jC4W-=`PRO7&PF7o;%L9$m1=Z{v6bc2DN+tdp zAIILJB5d6LB68Y#u}7-KEB(`W^$#U>-x$S_({+$ZrBJZBa=9D|l?rn9(V9~oSzoHL{Rt{R62ZzJtMDWcY`LsV20qNAg+iOqejx)vYK7;#MR#4(+Nx6*6f zkYw-X_XjAjTg|^bP=C^avW7vC-7tQC1~eOesgJ!&S-*Kt>I%ioGaH~$i(Gx!^&{U zEG&*-^gJq+p}F-kE-^b$!x(q+e-us7k?=Y%uLUlJ-_1Og(jF7P9U~|z@Kt(ETcgv z?vdobpn-?=+kfZ5BKHt=_pN`V`12aDb4_KKHG=a387q6E=3 + return current_app.extensions['migrate'].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace( + '%', '%%') + except AttributeError: + return str(get_engine().url).replace('%', '%%') + + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +config.set_main_option('sqlalchemy.url', get_engine_url()) +target_db = current_app.extensions['migrate'].db + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_metadata(): + if hasattr(target_db, 'metadatas'): + return target_db.metadatas[None] + return target_db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + conf_args = current_app.extensions['migrate'].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives + + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=get_metadata(), + **conf_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/31cd88d78cb4_innitial_commit.py b/migrations/versions/31cd88d78cb4_innitial_commit.py new file mode 100644 index 0000000..2a4a0d1 --- /dev/null +++ b/migrations/versions/31cd88d78cb4_innitial_commit.py @@ -0,0 +1,47 @@ +"""Innitial commit + +Revision ID: 31cd88d78cb4 +Revises: +Create Date: 2024-01-21 14:31:12.188455 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '31cd88d78cb4' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('session', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('authorized', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('user', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('first_name', sa.String(), nullable=True), + sa.Column('last_name', sa.String(), nullable=True), + sa.Column('phone', sa.String(), nullable=True), + sa.Column('username', sa.String(), nullable=True), + sa.Column('session_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['session_id'], ['session.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('phone'), + sa.UniqueConstraint('username') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('user') + op.drop_table('session') + # ### end Alembic commands ### diff --git a/migrations/versions/af22ea98e0da_deleted_json_filed.py b/migrations/versions/af22ea98e0da_deleted_json_filed.py new file mode 100644 index 0000000..38efed7 --- /dev/null +++ b/migrations/versions/af22ea98e0da_deleted_json_filed.py @@ -0,0 +1,42 @@ +"""Deleted json filed + +Revision ID: af22ea98e0da +Revises: 31cd88d78cb4 +Create Date: 2024-01-22 02:14:59.504400 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'af22ea98e0da' +down_revision = '31cd88d78cb4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('task_id', sa.String(), nullable=True), + sa.Column('name', sa.String(), nullable=False), + sa.Column('session_id', sa.Integer(), nullable=True), + sa.Column('status', sa.String(), nullable=False), + sa.Column('status_message', sa.String(), nullable=False), + sa.Column('type', sa.String(), nullable=False), + sa.Column('url', sa.String(), nullable=True), + sa.Column('message', sa.String(), nullable=True), + sa.Column('file', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['session_id'], ['session.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('task') + # ### end Alembic commands ### diff --git a/paper/__init__.py b/paper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper/client.py b/paper/client.py new file mode 100644 index 0000000..3ba49af --- /dev/null +++ b/paper/client.py @@ -0,0 +1,103 @@ +from loguru import logger + +from opentele.tl import TelegramClient +from opentele.api import API + +from telethon.tl.functions.channels import InviteToChannelRequest, JoinChannelRequest +from telethon.tl.functions.messages import AddChatUserRequest + +# Types +from telethon.hints import (Entity, EntityLike, MessageLike) +from telethon.types import Channel, Chat, User +from telethon.sessions.abstract import Session + +from paper.errors import * +from telethon.errors.rpcerrorlist import ( + ChatAdminRequiredError, + UserPrivacyRestrictedError, + UserNotMutualContactError, + ChannelPrivateError, + ChatWriteForbiddenError, + PeerFloodError, + FloodWaitError, + UsersTooMuchError, + UserChannelsTooMuchError, + UserIsBlockedError, + YouBlockedUserError +) + +class PaperClient(TelegramClient): + def __init__(self, session: str | Session) -> None: + api = API.TelegramAndroid.Generate("paper") + super().__init__(session, api) + + async def invite_self(self, group: Entity | EntityLike): + group = await self.__cast_to_entity(group) + + if isinstance(group, Channel): + await self(JoinChannelRequest(group)) + logger.info(f"Added self to a channel {group.title}") + + @logger.catch(reraise=True) + async def invite_user(self, user: Entity | EntityLike, group: Entity | EntityLike): + user, group = await self.__cast_to_entity(user), await self.__cast_to_entity(group) + + try: + if isinstance(group, Channel) and isinstance(user, User): + await self(InviteToChannelRequest(group, [user])) # type: ignore + logger.info(f"User {user} was added to channel {group.title}") + + elif isinstance(group, Chat) and isinstance(user, User): + await self(AddChatUserRequest(group, user, 30)) # type: ignore + logger.info(f"User {user} was added to chat {group.title}") + + else: + logger.warning(f"Can't determine group type for {group}. Skipping...") + + except (UserPrivacyRestrictedError, UserNotMutualContactError) as e: + raise UserPrivacyException(e) + + except (ChatWriteForbiddenError, ChannelPrivateError, ChatAdminRequiredError) as e: + raise AdminPrivilegesExceptions(e) + + except (PeerFloodError, FloodWaitError) as e: + raise FloodException(e) + + except (UserChannelsTooMuchError, UsersTooMuchError, UserIsBlockedError, YouBlockedUserError) as e: + raise IgnoreException(e) + + @logger.catch(reraise=True) + async def send_message(self, entity: Entity | EntityLike, message: str, file: str | None): + entity = await self.__cast_to_entity(entity) + + try: + return await super().send_message(entity, message=message, file=file) #type: ignore + + except (UserPrivacyRestrictedError, UserNotMutualContactError) as e: + raise UserPrivacyException(e) + + except (ChatWriteForbiddenError, ChannelPrivateError, ChatAdminRequiredError) as e: + raise AdminPrivilegesExceptions(e) + + except (PeerFloodError, FloodWaitError) as e: + raise FloodException(e) + + except (UserChannelsTooMuchError, UsersTooMuchError, UserIsBlockedError, YouBlockedUserError) as e: + raise IgnoreException(e) + + + @logger.catch(reraise=True) + async def get_participants(self, group: Entity | EntityLike, *args, **kwargs): + group = await self.__cast_to_entity(group) + + return await super().get_participants(group, *args, **kwargs) + + async def __cast_to_entity(self, entity: Entity | EntityLike) -> Entity: + if not isinstance(entity, Entity): + if hasattr(entity, "username"): + entity = await self.get_entity(entity.username) + else: + entity = await self.get_entity(entity) + + return entity # type: ignore + diff --git a/paper/errors/__init__.py b/paper/errors/__init__.py new file mode 100644 index 0000000..90e142e --- /dev/null +++ b/paper/errors/__init__.py @@ -0,0 +1,40 @@ +class UserPrivacyException(Exception): + """Occurs when user's privacy setting doesn't allow certain actions (e.g., inviting to a channel, sending a message)""" + def __init__(self, error: Exception, *args: object) -> None: + self.error = error + + self.message = f"Can't do action due to user's privacy settings" + super().__init__(self.message, *args) + +class AdminPrivilegesExceptions(Exception): + """Occurs when administrator privileges are required to perform a certain action (e.g., channel invitations, user parsing)""" + def __init__(self, error: Exception, *args: object) -> None: + self.error = error + + self.message = f"Administrator privileges are required to perform the action." + super().__init__(self.message, *args) + +class FloodException(Exception): + def __init__(self, error, *args: object): + self.error = error + self.seconds = self.error.seconds if hasattr(self.error, "seconds") else -1 #type: ignore + + super().__init__(*args) + +class IgnoreException(Exception): + def __init__(self, error, *args): + self.error = error + + super().__init__(*args) + +class TooMuchException(Exception): + def __init__(self, error, *args): + self.error = error + + super().__init__(*args) + +class NeedPasswordException(Exception): + def __init__(self, *args): + self.message = "Need password" + + super().__init__(self.message, *args) \ No newline at end of file diff --git a/paper/models.py b/paper/models.py new file mode 100644 index 0000000..c42101c --- /dev/null +++ b/paper/models.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + +from paper.utils.classes import DataclassBase + +dataclass(init=False) +class Message(DataclassBase): + text: str = '' + images: None | list[str] = None + + force_document: bool = False + + def exists(self): + return self.text or self.images \ No newline at end of file diff --git a/paper/parser.py b/paper/parser.py new file mode 100644 index 0000000..ef72bc3 --- /dev/null +++ b/paper/parser.py @@ -0,0 +1,92 @@ +import asyncio +import os + +from loguru import logger + +from paper.client import PaperClient +from paper.errors import IgnoreException, NeedPasswordException, UserPrivacyException +from paper.models import Message + + +class PaperParser: + def __init__(self, session: str) -> None: + if not session: + raise ValueError("Session name can't be None") + + self.client = PaperClient(session) + + self.users_to_delete = [] + + async def invite_users(self, users, group, task = None): + await self.client.invite_self(group) + + group_participants = await self.client.get_participants(group) + + if group_participants: + participants_usernames = [participant.username for participant in group_participants if participant.username] + users = filter(lambda user: user.username not in participants_usernames, users) + + for user in users: + try: + if task.is_aborted(): + return self.users_to_delete + + await self.client.invite_user(user, group) + + except (UserPrivacyException, IgnoreException) as e: + self.users_to_delete.append(user) + logger.exception(e) + logger.warning("Exception occurred. Skipping user...") + + except Exception as e: + logger.exception(e) + + finally: + if not task.is_aborted(): + await asyncio.sleep(50) # FIXME: Change to config value + + async def send_messages(self, users, message: str, file: str | None = None, task = None): + # TODO: Filter users, that already get this message + # for dialog in await self.client.get_dialogs(): + # if dialog.is_user: + # messages = tuple(filter(lambda chat_message: message.text == chat_message.text, await self.client.get_messages(dialog))) + # print(messages) + + for user in users: + try: + if task.is_aborted(): + return self.users_to_delete + + await self.client.send_message(user, message, file) + + except (UserPrivacyException, IgnoreException) as e: + self.users_to_delete.append(user) + logger.exception(e) + logger.warning("Exception occurred. Skipping user...") + + finally: + if not task.is_aborted(): + await asyncio.sleep(50) # FIXME: Change to config value + + async def get_participants(self, group): + await self.client.invite_self(group) + + return await self.client.get_participants(group) + + async def sign_in(self, phone: str, password: str | None = None, code: str | None = None, phone_hash: str | None = None, **kwargs): + if not await self.client.is_user_authorized(): + try: + return await self.client.sign_in(phone=phone, code=code, phone_code_hash=phone_hash) #type: ignore + except: + if not password: + raise NeedPasswordException() + + return await self.client.sign_in(password=password) + + async def __aenter__(self, *args, **kwargs): + await self.client.connect() + + return self + + async def __aexit__(self, *args, **kwargs): + await self.client.disconnect() \ No newline at end of file diff --git a/paper/utils/classes.py b/paper/utils/classes.py new file mode 100644 index 0000000..d7d25b5 --- /dev/null +++ b/paper/utils/classes.py @@ -0,0 +1,20 @@ +import dataclasses + + +@dataclasses.dataclass(init=False) +class DataclassBase: + """ + It just works™ + """ + + def __init__(self, **kwargs): + names = set([f.name for f in dataclasses.fields(self)]) + for k, v in kwargs.items(): + if k in names: + setattr(self, k, v) + + def dict(self): + return { + k: v.dict() if isinstance(v, DataclassBase) else v + for k, v in dataclasses.asdict(self).items() + } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ef94012 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,43 @@ +alembic==1.13.1 +amqp==5.2.0 +asgiref==3.7.2 +async-timeout==4.0.3 +billiard==4.2.0 +blinker==1.7.0 +celery==5.3.6 +click==8.1.7 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 +Flask==3.0.1 +Flask-Migrate==4.0.5 +Flask-SQLAlchemy==3.1.1 +greenlet==3.0.3 +gunicorn==21.2.0 +itsdangerous==2.1.2 +Jinja2==3.1.3 +kombu==5.3.5 +loguru==0.7.2 +Mako==1.3.0 +MarkupSafe==2.1.4 +opentele==1.15.1 +packaging==23.2 +prompt-toolkit==3.0.43 +psycopg2-binary==2.9.9 +pyaes==1.6.1 +pyasn1==0.5.1 +PyQt5==5.15.10 +PyQt5-Qt5==5.15.2 +PyQt5-sip==12.13.0 +python-dateutil==2.8.2 +redis==5.0.1 +rsa==4.9 +six==1.16.0 +SQLAlchemy==2.0.25 +Telethon==1.33.1 +TgCrypto==1.2.5 +typing_extensions==4.9.0 +tzdata==2023.4 +vine==5.1.0 +wcwidth==0.2.13 +Werkzeug==3.0.1