From 8dce3b1f62f6a24fa1ec560cac88ad8eaade078b Mon Sep 17 00:00:00 2001 From: winet Date: Thu, 8 Feb 2024 17:59:17 +1000 Subject: [PATCH 1/6] Fixed flood error endless sleep --- paper/parser.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/paper/parser.py b/paper/parser.py index ef72bc3..e4261b0 100644 --- a/paper/parser.py +++ b/paper/parser.py @@ -35,15 +35,11 @@ class PaperParser: except (UserPrivacyException, IgnoreException) as e: self.users_to_delete.append(user) - logger.exception(e) logger.warning("Exception occurred. Skipping user...") - except Exception as e: - logger.exception(e) - finally: if not task.is_aborted(): - await asyncio.sleep(50) # FIXME: Change to config value + await asyncio.sleep(70) # FIXME: Change to config value async def send_messages(self, users, message: str, file: str | None = None, task = None): # TODO: Filter users, that already get this message @@ -61,12 +57,11 @@ class PaperParser: except (UserPrivacyException, IgnoreException) as e: self.users_to_delete.append(user) - logger.exception(e) logger.warning("Exception occurred. Skipping user...") finally: if not task.is_aborted(): - await asyncio.sleep(50) # FIXME: Change to config value + await asyncio.sleep(70) # FIXME: Change to config value async def get_participants(self, group): await self.client.invite_self(group) From 94302d09e9e18cb1e0a2f818a87296fd1dd59efe Mon Sep 17 00:00:00 2001 From: winet Date: Thu, 8 Feb 2024 17:59:42 +1000 Subject: [PATCH 2/6] Added skipping of user with invalid username --- paper/client.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/paper/client.py b/paper/client.py index 07d365c..32ca1d1 100644 --- a/paper/client.py +++ b/paper/client.py @@ -23,7 +23,8 @@ from telethon.errors.rpcerrorlist import ( UsersTooMuchError, UserChannelsTooMuchError, UserIsBlockedError, - YouBlockedUserError + YouBlockedUserError, + UsernameInvalidError, ) class PaperClient(TelegramClient): @@ -93,11 +94,13 @@ class PaperClient(TelegramClient): return await super().get_participants(group, *args, **kwargs) async def __cast_to_entity(self, entity: Entity | EntityLike) -> Entity: - if not isinstance(entity, Entity): - if hasattr(entity, "username"): - entity = await self.get_entity(entity.username) - else: - entity = await self.get_entity(entity) - - return entity # type: ignore + try: + if not isinstance(entity, Entity): + if hasattr(entity, "username"): + entity = await self.get_entity(entity.username) + else: + entity = await self.get_entity(entity) + return entity # type: ignore + except UsernameInvalidError as e: + raise IgnoreException(e) From 81f97ee3d1a93a72210aaff82cfc95a7b3102613 Mon Sep 17 00:00:00 2001 From: winet Date: Thu, 8 Feb 2024 18:00:06 +1000 Subject: [PATCH 3/6] flood exception message --- paper/errors/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/paper/errors/__init__.py b/paper/errors/__init__.py index 90e142e..72c17f0 100644 --- a/paper/errors/__init__.py +++ b/paper/errors/__init__.py @@ -18,8 +18,9 @@ class FloodException(Exception): def __init__(self, error, *args: object): self.error = error self.seconds = self.error.seconds if hasattr(self.error, "seconds") else -1 #type: ignore - - super().__init__(*args) + self.message = f"Flood wait for {self.seconds}" if self.seconds > 0 else "Flood error" + + super().__init__(self.message, *args) class IgnoreException(Exception): def __init__(self, error, *args): From 557d19ea0dd4b281bc07f67e98d1250955cfcd51 Mon Sep 17 00:00:00 2001 From: winet Date: Thu, 8 Feb 2024 18:00:37 +1000 Subject: [PATCH 4/6] Changed way of changing task's message --- app/blueprints/api/tasks/handlers.py | 10 +++++----- app/blueprints/api/tasks/tasks.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/blueprints/api/tasks/handlers.py b/app/blueprints/api/tasks/handlers.py index d5d72fc..4243fa9 100644 --- a/app/blueprints/api/tasks/handlers.py +++ b/app/blueprints/api/tasks/handlers.py @@ -2,18 +2,18 @@ from app.extensions import db from app.models.task import Task def success_state(task_id): - task = Task.query.filter_by(task_id=task_id).one() + task = Task.query.get(task_id) task.status = "SUCCESS" - task.status_message = "Задача выполнена успешно" + task.status_message = "задача выполнена успешно" db.session.commit() def failure_state(task_id, exception): - task = Task.query.filter_by(task_id=task_id).one() + task = Task.query.get(task_id) task.status = "FAILURE" - task.status_message = f"Произошла ошибка: {str(exception)}" + task.status_message = f"произошла ошибка {str(exception)}" db.session.commit() @@ -22,6 +22,6 @@ def run_state(task_id, task_record_id): task.task_id = task_id task.status = "RUNNING" - task.status_message = "Задача запущена" + task.status_message = "задача запущена" db.session.commit() \ No newline at end of file diff --git a/app/blueprints/api/tasks/tasks.py b/app/blueprints/api/tasks/tasks.py index c4a3e5c..99526e0 100644 --- a/app/blueprints/api/tasks/tasks.py +++ b/app/blueprints/api/tasks/tasks.py @@ -31,11 +31,11 @@ def add_to_group_task(self, task_id): ) except Exception as e: - failure_state(self.request.id, e) + failure_state(task_id, e) raise e else: - success_state(self.request.id) + success_state(task_id) From 986341722fd551c17204b5959ecb8e8a6096839a Mon Sep 17 00:00:00 2001 From: winet Date: Sun, 11 Feb 2024 20:16:11 +1000 Subject: [PATCH 5/6] Show only authorized sessions --- app/blueprints/api/tasks/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/blueprints/api/tasks/routes.py b/app/blueprints/api/tasks/routes.py index 7ba6a79..e163b96 100644 --- a/app/blueprints/api/tasks/routes.py +++ b/app/blueprints/api/tasks/routes.py @@ -12,7 +12,7 @@ tasks = Blueprint("tasks", __name__, url_prefix="/tasks", template_folder="templ @tasks.route("/", methods=["GET"]) def get_tasks(): tasks = Task.query.all() - sessions = Session.query.all() + sessions = Session.query.filter_by(authorized=True).all() return render_template("tasks_cards.j2", tasks=tasks, sessions=sessions) From ac79aefa36d70b570a7fdcd2b758e2ffaa32f45d Mon Sep 17 00:00:00 2001 From: winet Date: Sun, 11 Feb 2024 22:22:18 +1000 Subject: [PATCH 6/6] sessions archive uploading handling --- .gitignore | 1 + app/blueprints/api/sessions/routes.py | 60 +++++++++++++++++++----- app/blueprints/frontend/templates/add.j2 | 9 ++-- config.py | 3 +- 4 files changed, 57 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 92ab989..01c2c5a 100644 --- a/.gitignore +++ b/.gitignore @@ -164,3 +164,4 @@ cython_debug/ *.session .vscode/settings.json *session_journal +uploads \ No newline at end of file diff --git a/app/blueprints/api/sessions/routes.py b/app/blueprints/api/sessions/routes.py index ce749e4..a4f56b5 100644 --- a/app/blueprints/api/sessions/routes.py +++ b/app/blueprints/api/sessions/routes.py @@ -1,16 +1,23 @@ -from flask import Blueprint, Response, jsonify, request +import os + +from flask import Blueprint, jsonify, request, current_app, redirect +from werkzeug.utils import secure_filename from app.extensions import db from app.models.session import Session -from app.models.task import Task from paper.errors import NeedPasswordException from paper.parser import PaperParser sessions = Blueprint("sessions", __name__, url_prefix="/sessions") -@sessions.route("/", methods=["POST"]) -async def create_session(**kwargs): - data = kwargs or request.json or request.form +ALLOWED_EXTENSIONS = ['zip'] + +def allowed_file(filename): + return '.' in filename and \ + filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + +async def create_session_form(): + data = request.json data["name"] = data["name"].replace("/", "").replace("\\", "") session = Session.query.filter_by(name=data.get("name")).first() @@ -54,9 +61,39 @@ async def create_session(**kwargs): finally: db.session.commit() - + return jsonify(response) +def create_session_from_file(): + archive = request.files['archive'] + + if archive.filename == '': + return 'Not Found', 404 + + if not allowed_file(archive.filename): + return 'File Extension Not Allowed', 406 + + session_upload_path = os.path.join(current_app.config['UPLOAD_DIR'], "sessions") + + if not os.path.exists(session_upload_path): + os.makedirs(session_upload_path) + + filename = secure_filename(archive.filename) + file_path = os.path.join(session_upload_path, filename) + + archive.save(file_path) + + return 'Ok', 200 + # return redirect('/') + +@sessions.route("/", methods=["POST"]) +async def create_session(): + files = request.files + if not 'archive' in files: + return await create_session_form() + else: + return create_session_from_file() + @sessions.route("/", methods=["GET"]) def get_sessions(**kwargs): return Session.query.filter_by(authorized=True).all() @@ -72,10 +109,11 @@ async def remove_session(id: int): try: await parser.client.log_out() + db.session.delete(session) + db.session.commit() + except Exception as e: - return Response(e, 500) + return str(e), 500 - db.session.delete(session) - db.session.commit() - - return Response('ok', status=200) \ No newline at end of file + return 'ok', 200 + \ No newline at end of file diff --git a/app/blueprints/frontend/templates/add.j2 b/app/blueprints/frontend/templates/add.j2 index e407810..9e91f09 100644 --- a/app/blueprints/frontend/templates/add.j2 +++ b/app/blueprints/frontend/templates/add.j2 @@ -91,10 +91,11 @@ PaperParser: Вход в аккаунт
-
- -
- + + +
+ +
diff --git a/config.py b/config.py index cddcf06..3f7ee55 100644 --- a/config.py +++ b/config.py @@ -13,8 +13,9 @@ class ProductionConfig: ) class DebugConfig: - SQLALCHEMY_DATABASE_URI = f"postgresql://postgres:123456789@localhost:5432/paper" + SQLALCHEMY_DATABASE_URI = f"postgresql://paper:123456789@localhost:5432/paper" SQLALCHEMY_TRACK_MODIFICATIONS = False + UPLOAD_DIR = 'uploads' CELERY = dict( broker_url="redis://localhost:6379/0", result_backend="redis://localhost:6379/0",