diff --git a/.gitignore b/.gitignore index 5d381cc..92ab989 100644 --- a/.gitignore +++ b/.gitignore @@ -160,3 +160,7 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +*.db +*.session +.vscode/settings.json +*session_journal diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..a125546 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,21 @@ +{ + "configurations": [ + { + "name": "Python: Flask", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "app", + "FLASK_DEBUG": "1" + }, + "args": [ + "run", + "--no-debugger", + "--no-reload", + ], + "jinja": true, + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ba897f4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11 + +COPY app/ /app/ +COPY migrations/ /migrations/ +COPY paper /paper/ + +COPY config.py config.py +COPY make_celery.py make_celery.py +COPY requirements.txt requirements.txt + +RUN pip install -r requirements.txt \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e6fade9 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,20 @@ +from flask import Flask + +from config import Config +from app.blueprints import blueprints +from app.extensions import db, migrate +from app.celery import celery_init_app + +def create_app(config_class=Config): + app = Flask(__name__) + app.config.from_object(config_class) + + db.init_app(app) + migrate.init_app(app) + + for blueprint in blueprints: + app.register_blueprint(blueprint) + + celery_init_app(app) + + return app \ No newline at end of file diff --git a/app/blueprints/__init__.py b/app/blueprints/__init__.py new file mode 100644 index 0000000..c7775ff --- /dev/null +++ b/app/blueprints/__init__.py @@ -0,0 +1,7 @@ +from .api.routes import api +from .frontend.routes import frontend + +blueprints = ( + api, + frontend, +) \ No newline at end of file diff --git a/app/blueprints/api/routes.py b/app/blueprints/api/routes.py new file mode 100644 index 0000000..0cd1db4 --- /dev/null +++ b/app/blueprints/api/routes.py @@ -0,0 +1,11 @@ +from flask import Blueprint + +from .tasks.routes import tasks +from .sessions.routes import sessions +from .users.routes import users + +api = Blueprint("api", __name__, url_prefix="/api") + +api.register_blueprint(users) +api.register_blueprint(tasks) +api.register_blueprint(sessions) \ No newline at end of file diff --git a/app/blueprints/api/sessions/routes.py b/app/blueprints/api/sessions/routes.py new file mode 100644 index 0000000..ce749e4 --- /dev/null +++ b/app/blueprints/api/sessions/routes.py @@ -0,0 +1,81 @@ +from flask import Blueprint, Response, jsonify, request + +from app.extensions import db +from app.models.session import Session +from app.models.task import Task +from paper.errors import NeedPasswordException +from paper.parser import PaperParser + +sessions = Blueprint("sessions", __name__, url_prefix="/sessions") + +@sessions.route("/", methods=["POST"]) +async def create_session(**kwargs): + data = kwargs or request.json or request.form + data["name"] = data["name"].replace("/", "").replace("\\", "") + + session = Session.query.filter_by(name=data.get("name")).first() + + if not session: + session = Session( + name=data.get("name"), + authorized=False + ) + + db.session.add(session) + + response = {"status": "ok"} + + async with PaperParser(data.get("name")) as parser: + try: + result = await parser.sign_in(**data) + + if hasattr(result, "phone_code_hash"): + response = { + "status": "error", + "message": "Need secret code from telegram", + "action": "code", + "phone_hash": result.phone_code_hash, + } + else: + session.authorized = True + + except NeedPasswordException: + response = { + "status": "error", + "message": "Need 2FA Password", + "action": "password", + } + + except Exception as e: + response = { + "status": "exception", + "message": str(e), + } + + finally: + db.session.commit() + + return jsonify(response) + +@sessions.route("/", methods=["GET"]) +def get_sessions(**kwargs): + return Session.query.filter_by(authorized=True).all() + +@sessions.route("/", methods=["DELETE"]) +async def remove_session(id: int): + session: Session = Session.query.get_or_404(id) + + if session.has_depending_tasks(): + return 'Есть зависимые задачи', 409 + + async with PaperParser(session.name) as parser: + try: + await parser.client.log_out() + + except Exception as e: + return Response(e, 500) + + db.session.delete(session) + db.session.commit() + + return Response('ok', status=200) \ No newline at end of file diff --git a/app/blueprints/api/tasks/handlers.py b/app/blueprints/api/tasks/handlers.py new file mode 100644 index 0000000..d5d72fc --- /dev/null +++ b/app/blueprints/api/tasks/handlers.py @@ -0,0 +1,27 @@ +from app.extensions import db +from app.models.task import Task + +def success_state(task_id): + task = Task.query.filter_by(task_id=task_id).one() + + task.status = "SUCCESS" + task.status_message = "Задача выполнена успешно" + + db.session.commit() + +def failure_state(task_id, exception): + task = Task.query.filter_by(task_id=task_id).one() + + task.status = "FAILURE" + task.status_message = f"Произошла ошибка: {str(exception)}" + + db.session.commit() + +def run_state(task_id, task_record_id): + task = Task.query.get(task_record_id) + + task.task_id = task_id + task.status = "RUNNING" + task.status_message = "Задача запущена" + + db.session.commit() \ No newline at end of file diff --git a/app/blueprints/api/tasks/routes.py b/app/blueprints/api/tasks/routes.py new file mode 100644 index 0000000..3f8a76a --- /dev/null +++ b/app/blueprints/api/tasks/routes.py @@ -0,0 +1,83 @@ +import uuid +from flask import Blueprint, render_template, request + +from app.extensions import db +from app.models.task import Task +from app.models.session import Session + +from .tasks import send_messages_task, add_to_group_task + +tasks = Blueprint("tasks", __name__, url_prefix="/tasks", template_folder="templates") + +@tasks.route("/", methods=["GET"]) +def get_tasks(): + tasks = Task.query.all() + + return render_template("tasks_cards.j2", tasks=tasks) + +@tasks.route("/", methods=["GET"]) +def get_task(task_id: int): + ... + +@tasks.route("/session/", methods=["POST"]) +def create_task(session_id: int): + info = request.form + session = Session.query.get_or_404(session_id) + task = Task( + name = str(uuid.uuid4()), + session = session, + status = "CREATED", + status_message = "Задача создана", + type = info.get('task'), + url = info.get('url'), + message = info.get('message'), + file = None + ) + + db.session.add(task) + db.session.commit() + + start_task(task.id) + + return 'Created', 200 + +@tasks.route("/", methods=["DELETE"]) +def delete_task(task_id: int): + task: Task = Task.query.get_or_404(task_id) + stop_task(task.id) + db.session.delete(task) + db.session.commit() + + return 'Deleted', 200 + +@tasks.route("//stop", methods=["PUT"]) +def stop_task(task_id: int): + task = Task.query.get_or_404(task_id) + fn = None + + match task.type: + case 'message': + fn = send_messages_task + + case 'add': + fn = add_to_group_task + + if task.task_id: + task_fn = fn.AsyncResult(task.task_id) + task_fn.abort() + + return 'Stopped', 200 + + +@tasks.route("//start", methods=["PUT"]) +def start_task(task_id: int): + task: Task = Task.query.get_or_404(task_id) + + match task.type: + case 'message': + send_messages_task.delay(task_id=task_id) + + case 'add': + add_to_group_task.delay(task_id=task_id) + + return 'Started', 200 \ No newline at end of file diff --git a/app/blueprints/api/tasks/tasks.py b/app/blueprints/api/tasks/tasks.py new file mode 100644 index 0000000..c4a3e5c --- /dev/null +++ b/app/blueprints/api/tasks/tasks.py @@ -0,0 +1,47 @@ +import time + +from celery import shared_task +import asyncio + +from paper.parser import PaperParser +from app.models.task import Task +from app.extensions import db +from .handlers import run_state, failure_state, success_state + +async def add_to_group(session, task, task_self): + async with PaperParser(session.name) as parser: + await parser.invite_users(session.users, task.url, task_self) + +async def sending_message(session, task, task_self): + async with PaperParser(session.name) as parser: + await parser.send_messages(session.users, task.message, task.file, task_self) + +@shared_task(bind=True) +def add_to_group_task(self, task_id): + try: + task: Task = Task.query.get(task_id) + run_state(self.request.id, task_id) + + session = task.session + + time.sleep(10) + + asyncio.run( + add_to_group(session, task, self) + ) + + except Exception as e: + failure_state(self.request.id, e) + raise e + + else: + success_state(self.request.id) + + + +@shared_task(bind=True) +def send_messages_task(self, task_id): + task: Task = Task.query.get(task_id) + + session = task.session + users = session.users \ No newline at end of file diff --git a/app/blueprints/api/tasks/templates/tasks_cards.j2 b/app/blueprints/api/tasks/templates/tasks_cards.j2 new file mode 100644 index 0000000..d102e0e --- /dev/null +++ b/app/blueprints/api/tasks/templates/tasks_cards.j2 @@ -0,0 +1,34 @@ +{% for task in tasks %} +
+
+
+

{{ task.name }}

+
    +
  • Статус: {{ task.status }}
    • +
  • Сообщение: {{ task.status_message }}
    • +
  • Действие: {{ task.type }}
    • +
  • Сессия: {{ task.session.name }}
    • +
+
+
+ + + +
+
+
+{% else %} + Здесь ничего нет. +{% endfor %} \ No newline at end of file diff --git a/app/blueprints/api/users/routes.py b/app/blueprints/api/users/routes.py new file mode 100644 index 0000000..3f85d93 --- /dev/null +++ b/app/blueprints/api/users/routes.py @@ -0,0 +1,58 @@ +from flask import Blueprint, render_template, request + +from app.models.session import Session +from app.models.user import User +from app.extensions import db +from paper.parser import PaperParser + + +users = Blueprint("users", __name__, url_prefix="/users", template_folder="templates") + + +@users.route("/parse/", methods=["POST"]) +async def parse_users(session_id: int): + session = Session.query.get_or_404(session_id) + + data = request.form + + async with PaperParser(session.name) as parser: + users = await parser.get_participants(data.get("group")) + + for user in users: + if not user.username: + continue + + exists = db.session.query(User.id).filter_by(username=user.username).first() + if not exists: + db.session.add( + User( + first_name=user.first_name, + last_name=user.last_name, + username=user.username, + phone=user.phone, + session=session, + ) + ) + + db.session.commit() + + return render_template("user_cards.j2", users=session.users) + + +@users.route("/", methods=["GET"]) +def get_users(session_id: int): + users = Session.query.get_or_404(session_id).users + + return render_template("user_cards.j2", users=users) + +@users.route("/", methods=["DELETE"]) +def delete_users(session_id): + session: Session = Session.query.get_or_404(session_id) + + if session.has_depending_tasks(): + return 'Есть зависимые задачи', 409 + + User.query.filter_by(session=session).delete() + db.session.commit() + + return render_template("user_cards.j2", users=[]) \ No newline at end of file diff --git a/app/blueprints/api/users/templates/user_cards.j2 b/app/blueprints/api/users/templates/user_cards.j2 new file mode 100644 index 0000000..56a7c8e --- /dev/null +++ b/app/blueprints/api/users/templates/user_cards.j2 @@ -0,0 +1,18 @@ +{% for user in users %} +
+
+
+

{{ user.first_name }}

+
+ +
+
+{% else %} + Здесь ничего нет. +{% endfor %} \ No newline at end of file diff --git a/app/blueprints/frontend/routes.py b/app/blueprints/frontend/routes.py new file mode 100644 index 0000000..b22e35b --- /dev/null +++ b/app/blueprints/frontend/routes.py @@ -0,0 +1,23 @@ +from flask import Blueprint, render_template +from app.blueprints.api.sessions.routes import get_sessions +from app.blueprints.api.users.routes import get_users +from app.blueprints.api.tasks.routes import get_tasks + +frontend = Blueprint("frontend", __name__, url_prefix="/", template_folder="templates", static_folder="static", static_url_path="/static/frontend") + +@frontend.route("/") +def index(): + sessions = get_sessions() + return render_template("index.j2", sessions=sessions) + +@frontend.route("/add") +def add(): + return render_template("add.j2") + +@frontend.route("/tasks") +def tasks(): + return render_template("tasks.j2", tasks_template=get_tasks()) + +@frontend.route("/parse/") +def parse(id: int): + return render_template("parse.j2", session_id=id, users_template=get_users(id)) \ No newline at end of file diff --git a/app/blueprints/frontend/static/add.js b/app/blueprints/frontend/static/add.js new file mode 100644 index 0000000..e975b83 --- /dev/null +++ b/app/blueprints/frontend/static/add.js @@ -0,0 +1,115 @@ +const API_URL = window.location.origin; + +const nameInput = document.querySelector("#input-name"); +const phoneInput = document.querySelector("#input-phone"); +const addSessionForm = document.querySelector("#add-session-form"); + +const openFileButton = document.querySelector("#open-file-btn"); + +const secretCodeModal = new bootstrap.Modal('#secretCodeModal'); +const secretCodeForm = document.querySelector("#secret-code-form"); +const secretCodeInput = document.querySelector("#input-secret-code"); + +const passwordModal = new bootstrap.Modal('#passwordModal'); +const passwordForm = document.querySelector("#password-form"); +const passwordInput = document.querySelector("#input-password"); + +let phoneHash = ""; + +const handleCreation = async (e) => { + e.preventDefault(); + + if (phoneInput.value == "" || nameInput.value == "") { + alert("Пожалуйста, заполните все поля."); + return; + } + + const data = getAuthData(); + + res = await fetch(`${API_URL}/api/sessions/`, { + method: "POST", + body: JSON.stringify(data), + headers: { + "Content-Type": "application/json", + } + }); + + result = await res.json(); + + switch(result.status) { + case "ok": + window.location = "/"; + break; + + case "error": + handleError(result); + break; + + case "exception": + alert(`Ошибка: ${result.message}.`); + break; + + default: + alert(`Неизвестная ошибка. Проверьте логи`); + } +} + +const getAuthData = () => { + const inputFields = [passwordInput, secretCodeInput, nameInput, phoneInput] + data = {} + inputFields.forEach((el) => { + if (el.value !== "") { + data[el.attributes.name.value] = el.value.replace(/ /g, ''); + } + }) + + if (phoneHash !== "") + data["phone_hash"] = phoneHash; + + return data; +} + +const handleRequest = async (fn) => { +} + +const handleError = (result) => { + switch(result.action) { + case "code": + phoneHash = result.phone_hash; + secretCodeModal.show(); + break; + case "password": + passwordModal.show(); + break; + } +} + +addSessionForm.addEventListener("submit", handleCreation); +secretCodeForm.addEventListener("submit", handleCreation); +passwordForm.addEventListener("submit", handleCreation); + + +// openFileButton.addEventListener("click", handleOpening); + +function handleOpening() { + let input = document.createElement('input'); + input.type = 'file'; + input.accept = ".zip"; + input.onchange = async () => { + // you can use this method to get file and perform respective operations + let archive = Array.from(input.files)[0]; + let formData = new FormData(); + + formData.append("archive", archive); + response = await fetch(`${API_URL}/api/save_session_file.php`, { method: "POST", body: formData }); + + if (response.status == "201") { + window.location = "/" + } else { + const error_text = await response.text(); + const error_status = await response.status; + alert(`${error_status}: ${error_text}`); + } + }; + input.click(); +} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/add.j2 b/app/blueprints/frontend/templates/add.j2 new file mode 100644 index 0000000..e407810 --- /dev/null +++ b/app/blueprints/frontend/templates/add.j2 @@ -0,0 +1,105 @@ +{% extends "base.j2" %} + +{% block title %} +PaperParser: Вход в аккаунт +{% endblock title %} + +{% block main %} +
+ + + + + + +
+ + + +
+ +
+
+
+ + +
+
+
+
+ + +
+
+
+
+ + +
+
+
+ +
+ +
+
+
+{% endblock main %} + +{% block scripts %} + +{% endblock scripts %} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/base.j2 b/app/blueprints/frontend/templates/base.j2 new file mode 100644 index 0000000..c49d152 --- /dev/null +++ b/app/blueprints/frontend/templates/base.j2 @@ -0,0 +1,64 @@ + + + + + {% block title %}{% endblock %} + + + + + + + + + + + +
+ + +
+
+ {% block main %} + + {% endblock %} +
+ + + + + + + + + + + + + {% block scripts %} + {% endblock %} + + \ No newline at end of file diff --git a/app/blueprints/frontend/templates/index.j2 b/app/blueprints/frontend/templates/index.j2 new file mode 100644 index 0000000..5eb8117 --- /dev/null +++ b/app/blueprints/frontend/templates/index.j2 @@ -0,0 +1,38 @@ +{% extends "base.j2" %} + +{% block title %} +PaperParser: Главная +{% endblock title %} + +{% block main %} +
+
+ + {% for session in sessions %} +
+
+
+

{{ session.name }}

+
+
+ + Использовать + + + +
+
+
+ {% endfor %} + +
+
+{% endblock main %} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/parse.j2 b/app/blueprints/frontend/templates/parse.j2 new file mode 100644 index 0000000..bbdc5d5 --- /dev/null +++ b/app/blueprints/frontend/templates/parse.j2 @@ -0,0 +1,101 @@ +{% extends "base.j2" %} + +{% block title %} +PaperParser: Парсинг +{% endblock title %} + +{% block main %} +
+ +
+
+

Действия

+
+
+
+ +
+
+
+
+ + +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+ +
+
+ +
+
+
+ +
+
+
+
+ + +
+
+ +
+
+
+
+
+
+ + +
+
+
+
+

Пользователи

+ +
+
+ + +
+ +
+
+
+
+ {{ users_template }} +
+
+
+{% endblock main %} \ No newline at end of file diff --git a/app/blueprints/frontend/templates/tasks.j2 b/app/blueprints/frontend/templates/tasks.j2 new file mode 100644 index 0000000..27f6548 --- /dev/null +++ b/app/blueprints/frontend/templates/tasks.j2 @@ -0,0 +1,16 @@ +{% extends "base.j2" %} + +{% block title %} + PaperParser: Заадчи +{% endblock title %} + +{% block main %} +
+
+

Задачи

+
+
+
+ {{ tasks_template }} +
+{% endblock main %} \ No newline at end of file diff --git a/app/celery.py b/app/celery.py new file mode 100644 index 0000000..c5cdd85 --- /dev/null +++ b/app/celery.py @@ -0,0 +1,18 @@ +from flask import Flask +from celery import Celery +from celery.contrib.abortable import AbortableTask +from app.blueprints.api.tasks.handlers import * + +def celery_init_app(app: Flask) -> Celery: + class FlaskTask(AbortableTask): + def __call__(self, *args: object, **kwargs: object) -> object: + with app.app_context(): + return self.run(*args, **kwargs) + + celery_app = Celery(app.name, task_cls=FlaskTask) + + celery_app.config_from_object(app.config["CELERY"]) + celery_app.set_default() + app.extensions["celery"] = celery_app + return celery_app + \ No newline at end of file diff --git a/app/extensions.py b/app/extensions.py new file mode 100644 index 0000000..8f3dacb --- /dev/null +++ b/app/extensions.py @@ -0,0 +1,5 @@ +from flask_sqlalchemy import SQLAlchemy +from flask_migrate import Migrate + +db = SQLAlchemy() +migrate = Migrate(db=db) \ No newline at end of file diff --git a/app/models/session.py b/app/models/session.py new file mode 100644 index 0000000..900c8e7 --- /dev/null +++ b/app/models/session.py @@ -0,0 +1,24 @@ +from typing import List +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.models.task import Task +from app.models.user import User +from app.extensions import db + +class Session(db.Model): + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(unique=True) + + authorized: Mapped[bool] + users: Mapped[List[User]] = relationship("User", back_populates="session") + + tasks: Mapped[List[Task]] = relationship("Task", back_populates="session") + + def has_depending_tasks(self): + tasks = Task.query \ + .filter_by(session_id=self.id) \ + .filter((Task.status == "RUNNING") | (Task.status == "RUNNING")) \ + .all() + + return len(tasks) > 0 + \ No newline at end of file diff --git a/app/models/task.py b/app/models/task.py new file mode 100644 index 0000000..22676d3 --- /dev/null +++ b/app/models/task.py @@ -0,0 +1,20 @@ +from sqlalchemy import Integer +from sqlalchemy.schema import Column, ForeignKey +from sqlalchemy.orm import Mapped, mapped_column, relationship +from app.extensions import db + +class Task(db.Model): + id: Mapped[int] = mapped_column(primary_key=True) + task_id: Mapped[str] = mapped_column(nullable=True) + + name: Mapped[str] = mapped_column(unique=True) + + session_id = Column(Integer, ForeignKey("session.id")) + session: Mapped['Session'] = relationship("Session", back_populates="tasks") + status: Mapped[str] + status_message: Mapped[str] + type: Mapped[str] + + url: Mapped[str] = mapped_column(nullable=True) + message: Mapped[str] = mapped_column(nullable=True) + file: Mapped[str] = mapped_column(nullable=True) \ No newline at end of file diff --git a/app/models/user.py b/app/models/user.py new file mode 100644 index 0000000..510b00b --- /dev/null +++ b/app/models/user.py @@ -0,0 +1,19 @@ +from sqlalchemy import Integer +from sqlalchemy.schema import ( + Column, + ForeignKey, +) +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.extensions import db + +class User(db.Model): + id: Mapped[int] = mapped_column(primary_key=True) + first_name: Mapped[str] = mapped_column(nullable=True) + last_name: Mapped[str] = mapped_column(nullable=True) + + phone: Mapped[str] = mapped_column(nullable=True, unique=True) + username: Mapped[str] = mapped_column(nullable=True, unique=True) + + session_id = Column(Integer, ForeignKey("session.id")) + session: Mapped["Session"] = relationship("Session", back_populates="users") \ No newline at end of file diff --git a/app/static/favicon.ico b/app/static/favicon.ico new file mode 100644 index 0000000..d880527 Binary files /dev/null and b/app/static/favicon.ico differ diff --git a/app/utils/__init__.py b/app/utils/__init__.py new file mode 100644 index 0000000..9c3c733 --- /dev/null +++ b/app/utils/__init__.py @@ -0,0 +1,15 @@ +import asyncio +import functools + +def to_sync_task(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + loop = asyncio.get_event_loop() + except RuntimeError: + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + loop.run_forever() + return loop.run_until_complete(func(*args, **kwargs)) + + return wrapper \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..498564d --- /dev/null +++ b/config.py @@ -0,0 +1,14 @@ +import os + +basedir = os.path.abspath(os.path.dirname(__file__)) + +class Config: + SECRET_KEY = os.environ.get('SECRET_KEY') + SQLALCHEMY_DATABASE_URI = f"postgresql://{os.environ.get('POSTGRES_USER')}:{os.environ.get('POSTGRES_PASSWORD')}@postgres/{os.environ.get('POSTGRES_DB')}" + SQLALCHEMY_TRACK_MODIFICATIONS = False + CELERY = dict( + broker_url="redis://redis/0", + result_backend="redis://redis/0", + task_ignore_result=True, + ) + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b616a92 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,29 @@ +services: + redis: + image: redis:7 + + postgres: + env_file: + - .env + image: postgres:16 + + celery: + build: + context: . + env_file: + - .env + command: celery -A make_celery worker --loglevel INFO + depends_on: + - redis + + web: + build: + context: . + env_file: + - .env + command: gunicorn --bind 0.0.0.0:5000 "app:create_app()" + ports: + - 5000:5000 + depends_on: + - celery + - postgres \ No newline at end of file diff --git a/example.env b/example.env new file mode 100644 index 0000000..784271a --- /dev/null +++ b/example.env @@ -0,0 +1,4 @@ +FLASK_APP= +POSTGRES_USER= +POSTGRES_PASSWORD= +POSTGRES_DB= \ No newline at end of file diff --git a/make_celery.py b/make_celery.py new file mode 100644 index 0000000..dc976ce --- /dev/null +++ b/make_celery.py @@ -0,0 +1,4 @@ +from app import create_app + +flask_app = create_app() +celery_app = flask_app.extensions['celery'] \ No newline at end of file diff --git a/migrations/README b/migrations/README new file mode 100644 index 0000000..0e04844 --- /dev/null +++ b/migrations/README @@ -0,0 +1 @@ +Single-database configuration for Flask. diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 0000000..ec9d45c --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,50 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic,flask_migrate + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[logger_flask_migrate] +level = INFO +handlers = +qualname = flask_migrate + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..4c97092 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,113 @@ +import logging +from logging.config import fileConfig + +from flask import current_app + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + + +def get_engine(): + try: + # this works with Flask-SQLAlchemy<3 and Alchemical + return current_app.extensions['migrate'].db.get_engine() + except (TypeError, AttributeError): + # this works with Flask-SQLAlchemy>=3 + return current_app.extensions['migrate'].db.engine + + +def get_engine_url(): + try: + return get_engine().url.render_as_string(hide_password=False).replace( + '%', '%%') + except AttributeError: + return str(get_engine().url).replace('%', '%%') + + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +config.set_main_option('sqlalchemy.url', get_engine_url()) +target_db = current_app.extensions['migrate'].db + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def get_metadata(): + if hasattr(target_db, 'metadatas'): + return target_db.metadatas[None] + return target_db.metadata + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=get_metadata(), literal_binds=True + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + conf_args = current_app.extensions['migrate'].configure_args + if conf_args.get("process_revision_directives") is None: + conf_args["process_revision_directives"] = process_revision_directives + + connectable = get_engine() + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=get_metadata(), + **conf_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/31cd88d78cb4_innitial_commit.py b/migrations/versions/31cd88d78cb4_innitial_commit.py new file mode 100644 index 0000000..2a4a0d1 --- /dev/null +++ b/migrations/versions/31cd88d78cb4_innitial_commit.py @@ -0,0 +1,47 @@ +"""Innitial commit + +Revision ID: 31cd88d78cb4 +Revises: +Create Date: 2024-01-21 14:31:12.188455 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '31cd88d78cb4' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('session', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('authorized', sa.Boolean(), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + op.create_table('user', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('first_name', sa.String(), nullable=True), + sa.Column('last_name', sa.String(), nullable=True), + sa.Column('phone', sa.String(), nullable=True), + sa.Column('username', sa.String(), nullable=True), + sa.Column('session_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint(['session_id'], ['session.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('phone'), + sa.UniqueConstraint('username') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('user') + op.drop_table('session') + # ### end Alembic commands ### diff --git a/migrations/versions/af22ea98e0da_deleted_json_filed.py b/migrations/versions/af22ea98e0da_deleted_json_filed.py new file mode 100644 index 0000000..38efed7 --- /dev/null +++ b/migrations/versions/af22ea98e0da_deleted_json_filed.py @@ -0,0 +1,42 @@ +"""Deleted json filed + +Revision ID: af22ea98e0da +Revises: 31cd88d78cb4 +Create Date: 2024-01-22 02:14:59.504400 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'af22ea98e0da' +down_revision = '31cd88d78cb4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('task', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('task_id', sa.String(), nullable=True), + sa.Column('name', sa.String(), nullable=False), + sa.Column('session_id', sa.Integer(), nullable=True), + sa.Column('status', sa.String(), nullable=False), + sa.Column('status_message', sa.String(), nullable=False), + sa.Column('type', sa.String(), nullable=False), + sa.Column('url', sa.String(), nullable=True), + sa.Column('message', sa.String(), nullable=True), + sa.Column('file', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['session_id'], ['session.id'], ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('task') + # ### end Alembic commands ### diff --git a/paper/__init__.py b/paper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/paper/client.py b/paper/client.py new file mode 100644 index 0000000..3ba49af --- /dev/null +++ b/paper/client.py @@ -0,0 +1,103 @@ +from loguru import logger + +from opentele.tl import TelegramClient +from opentele.api import API + +from telethon.tl.functions.channels import InviteToChannelRequest, JoinChannelRequest +from telethon.tl.functions.messages import AddChatUserRequest + +# Types +from telethon.hints import (Entity, EntityLike, MessageLike) +from telethon.types import Channel, Chat, User +from telethon.sessions.abstract import Session + +from paper.errors import * +from telethon.errors.rpcerrorlist import ( + ChatAdminRequiredError, + UserPrivacyRestrictedError, + UserNotMutualContactError, + ChannelPrivateError, + ChatWriteForbiddenError, + PeerFloodError, + FloodWaitError, + UsersTooMuchError, + UserChannelsTooMuchError, + UserIsBlockedError, + YouBlockedUserError +) + +class PaperClient(TelegramClient): + def __init__(self, session: str | Session) -> None: + api = API.TelegramAndroid.Generate("paper") + super().__init__(session, api) + + async def invite_self(self, group: Entity | EntityLike): + group = await self.__cast_to_entity(group) + + if isinstance(group, Channel): + await self(JoinChannelRequest(group)) + logger.info(f"Added self to a channel {group.title}") + + @logger.catch(reraise=True) + async def invite_user(self, user: Entity | EntityLike, group: Entity | EntityLike): + user, group = await self.__cast_to_entity(user), await self.__cast_to_entity(group) + + try: + if isinstance(group, Channel) and isinstance(user, User): + await self(InviteToChannelRequest(group, [user])) # type: ignore + logger.info(f"User {user} was added to channel {group.title}") + + elif isinstance(group, Chat) and isinstance(user, User): + await self(AddChatUserRequest(group, user, 30)) # type: ignore + logger.info(f"User {user} was added to chat {group.title}") + + else: + logger.warning(f"Can't determine group type for {group}. Skipping...") + + except (UserPrivacyRestrictedError, UserNotMutualContactError) as e: + raise UserPrivacyException(e) + + except (ChatWriteForbiddenError, ChannelPrivateError, ChatAdminRequiredError) as e: + raise AdminPrivilegesExceptions(e) + + except (PeerFloodError, FloodWaitError) as e: + raise FloodException(e) + + except (UserChannelsTooMuchError, UsersTooMuchError, UserIsBlockedError, YouBlockedUserError) as e: + raise IgnoreException(e) + + @logger.catch(reraise=True) + async def send_message(self, entity: Entity | EntityLike, message: str, file: str | None): + entity = await self.__cast_to_entity(entity) + + try: + return await super().send_message(entity, message=message, file=file) #type: ignore + + except (UserPrivacyRestrictedError, UserNotMutualContactError) as e: + raise UserPrivacyException(e) + + except (ChatWriteForbiddenError, ChannelPrivateError, ChatAdminRequiredError) as e: + raise AdminPrivilegesExceptions(e) + + except (PeerFloodError, FloodWaitError) as e: + raise FloodException(e) + + except (UserChannelsTooMuchError, UsersTooMuchError, UserIsBlockedError, YouBlockedUserError) as e: + raise IgnoreException(e) + + + @logger.catch(reraise=True) + async def get_participants(self, group: Entity | EntityLike, *args, **kwargs): + group = await self.__cast_to_entity(group) + + return await super().get_participants(group, *args, **kwargs) + + async def __cast_to_entity(self, entity: Entity | EntityLike) -> Entity: + if not isinstance(entity, Entity): + if hasattr(entity, "username"): + entity = await self.get_entity(entity.username) + else: + entity = await self.get_entity(entity) + + return entity # type: ignore + diff --git a/paper/errors/__init__.py b/paper/errors/__init__.py new file mode 100644 index 0000000..90e142e --- /dev/null +++ b/paper/errors/__init__.py @@ -0,0 +1,40 @@ +class UserPrivacyException(Exception): + """Occurs when user's privacy setting doesn't allow certain actions (e.g., inviting to a channel, sending a message)""" + def __init__(self, error: Exception, *args: object) -> None: + self.error = error + + self.message = f"Can't do action due to user's privacy settings" + super().__init__(self.message, *args) + +class AdminPrivilegesExceptions(Exception): + """Occurs when administrator privileges are required to perform a certain action (e.g., channel invitations, user parsing)""" + def __init__(self, error: Exception, *args: object) -> None: + self.error = error + + self.message = f"Administrator privileges are required to perform the action." + super().__init__(self.message, *args) + +class FloodException(Exception): + def __init__(self, error, *args: object): + self.error = error + self.seconds = self.error.seconds if hasattr(self.error, "seconds") else -1 #type: ignore + + super().__init__(*args) + +class IgnoreException(Exception): + def __init__(self, error, *args): + self.error = error + + super().__init__(*args) + +class TooMuchException(Exception): + def __init__(self, error, *args): + self.error = error + + super().__init__(*args) + +class NeedPasswordException(Exception): + def __init__(self, *args): + self.message = "Need password" + + super().__init__(self.message, *args) \ No newline at end of file diff --git a/paper/models.py b/paper/models.py new file mode 100644 index 0000000..c42101c --- /dev/null +++ b/paper/models.py @@ -0,0 +1,13 @@ +from dataclasses import dataclass + +from paper.utils.classes import DataclassBase + +dataclass(init=False) +class Message(DataclassBase): + text: str = '' + images: None | list[str] = None + + force_document: bool = False + + def exists(self): + return self.text or self.images \ No newline at end of file diff --git a/paper/parser.py b/paper/parser.py new file mode 100644 index 0000000..ef72bc3 --- /dev/null +++ b/paper/parser.py @@ -0,0 +1,92 @@ +import asyncio +import os + +from loguru import logger + +from paper.client import PaperClient +from paper.errors import IgnoreException, NeedPasswordException, UserPrivacyException +from paper.models import Message + + +class PaperParser: + def __init__(self, session: str) -> None: + if not session: + raise ValueError("Session name can't be None") + + self.client = PaperClient(session) + + self.users_to_delete = [] + + async def invite_users(self, users, group, task = None): + await self.client.invite_self(group) + + group_participants = await self.client.get_participants(group) + + if group_participants: + participants_usernames = [participant.username for participant in group_participants if participant.username] + users = filter(lambda user: user.username not in participants_usernames, users) + + for user in users: + try: + if task.is_aborted(): + return self.users_to_delete + + await self.client.invite_user(user, group) + + except (UserPrivacyException, IgnoreException) as e: + self.users_to_delete.append(user) + logger.exception(e) + logger.warning("Exception occurred. Skipping user...") + + except Exception as e: + logger.exception(e) + + finally: + if not task.is_aborted(): + await asyncio.sleep(50) # FIXME: Change to config value + + async def send_messages(self, users, message: str, file: str | None = None, task = None): + # TODO: Filter users, that already get this message + # for dialog in await self.client.get_dialogs(): + # if dialog.is_user: + # messages = tuple(filter(lambda chat_message: message.text == chat_message.text, await self.client.get_messages(dialog))) + # print(messages) + + for user in users: + try: + if task.is_aborted(): + return self.users_to_delete + + await self.client.send_message(user, message, file) + + except (UserPrivacyException, IgnoreException) as e: + self.users_to_delete.append(user) + logger.exception(e) + logger.warning("Exception occurred. Skipping user...") + + finally: + if not task.is_aborted(): + await asyncio.sleep(50) # FIXME: Change to config value + + async def get_participants(self, group): + await self.client.invite_self(group) + + return await self.client.get_participants(group) + + async def sign_in(self, phone: str, password: str | None = None, code: str | None = None, phone_hash: str | None = None, **kwargs): + if not await self.client.is_user_authorized(): + try: + return await self.client.sign_in(phone=phone, code=code, phone_code_hash=phone_hash) #type: ignore + except: + if not password: + raise NeedPasswordException() + + return await self.client.sign_in(password=password) + + async def __aenter__(self, *args, **kwargs): + await self.client.connect() + + return self + + async def __aexit__(self, *args, **kwargs): + await self.client.disconnect() \ No newline at end of file diff --git a/paper/utils/classes.py b/paper/utils/classes.py new file mode 100644 index 0000000..d7d25b5 --- /dev/null +++ b/paper/utils/classes.py @@ -0,0 +1,20 @@ +import dataclasses + + +@dataclasses.dataclass(init=False) +class DataclassBase: + """ + It just works™ + """ + + def __init__(self, **kwargs): + names = set([f.name for f in dataclasses.fields(self)]) + for k, v in kwargs.items(): + if k in names: + setattr(self, k, v) + + def dict(self): + return { + k: v.dict() if isinstance(v, DataclassBase) else v + for k, v in dataclasses.asdict(self).items() + } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ef94012 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,43 @@ +alembic==1.13.1 +amqp==5.2.0 +asgiref==3.7.2 +async-timeout==4.0.3 +billiard==4.2.0 +blinker==1.7.0 +celery==5.3.6 +click==8.1.7 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 +Flask==3.0.1 +Flask-Migrate==4.0.5 +Flask-SQLAlchemy==3.1.1 +greenlet==3.0.3 +gunicorn==21.2.0 +itsdangerous==2.1.2 +Jinja2==3.1.3 +kombu==5.3.5 +loguru==0.7.2 +Mako==1.3.0 +MarkupSafe==2.1.4 +opentele==1.15.1 +packaging==23.2 +prompt-toolkit==3.0.43 +psycopg2-binary==2.9.9 +pyaes==1.6.1 +pyasn1==0.5.1 +PyQt5==5.15.10 +PyQt5-Qt5==5.15.2 +PyQt5-sip==12.13.0 +python-dateutil==2.8.2 +redis==5.0.1 +rsa==4.9 +six==1.16.0 +SQLAlchemy==2.0.25 +Telethon==1.33.1 +TgCrypto==1.2.5 +typing_extensions==4.9.0 +tzdata==2023.4 +vine==5.1.0 +wcwidth==0.2.13 +Werkzeug==3.0.1