collection support of tasks

app/blueprints/api/collections/routes.py

@@ -5,6 +5,26 @@ from app.extensions import db
 
 collections = Blueprint("collections", __name__, url_prefix="/collections", template_folder="templates")
 
+@collections.route('/', methods=["POST"])
+def create_collection():
+    data = request.form
+    name = data.get('collection_name')
+    
+    if not name:
+        return render_template('collections/create_form.j2', collections_error='Введите название базы!')
+    
+    if Collection.exists(name):
+        return render_template('collections/create_form.j2', collections_error='База уже существует!')
+    
+    swap = 'afterbegin' if len(Collection.query.all()) > 0 else 'innerHTML'
+    
+    collection = Collection(name=name)
+    
+    db.session.add(collection)
+    db.session.commit()
+    
+    return f"<option value=\"{collection.id}\" selected>{ collection.name }</option>", 200, {'Hx-Reswap': swap, 'HX-Retarget': '#collection-select'}
+
 @collections.route("/<int:id>", methods=["DELETE"])
 def delete_collection(id: int):
     
@@ -18,4 +38,4 @@ def delete_collection(id: int):
     if length <= 0:
         return "<small class=\"form-text\">Здесь ничего нет</small>", 200
     
-    return "", 204
+    return "", 200

app/blueprints/api/collections/templates/collections/card.j2

@@ -1,18 +1,20 @@
-<div data-collection-id="{{ collection.id }}" class="card">
-    <div class="card-header"><h5>{{ collection.name }}</h5></div>
-    <ul class="list-group list-group-flush">
-        <li class="list-group-item">Кол-во пользрвателей: {{ collection.users|length }}</li>
-    </ul>
-    <form action="/api/collections/{{ collection.id }}" method="DELETE" class="card-footer px-2 pt-2">
-        <button
-            type="submit"
-            hx-delete="/api/collections/{{ collection.id }}"
-            hx-swap="outerHTML"
-            hx-target='[data-collection-id="{{ collection.id }}"]'
-            hx-confirm="Вы уверены, что хотите удалить коллекцию?"
-            class="btn btn-outline-danger"
-        >
-            Удалить
-        </button>
-    </form>
+<div data-collection-id="{{ collection.id }}" class="col mb-3">
+    <div class="card">
+        <div class="card-header"><h5>{{ collection.name }}</h5></div>
+        <ul class="list-group list-group-flush">
+            <li class="list-group-item">Кол-во пользрвателей: {{ collection.users|length }}</li>
+        </ul>
+        <form action="/api/collections/{{ collection.id }}" method="DELETE" class="card-footer px-2 pt-2">
+            <button
+                type="submit"
+                hx-delete="/api/collections/{{ collection.id }}"
+                hx-swap="outerHTML"
+                hx-target='[data-collection-id="{{ collection.id }}"]'
+                hx-confirm="Вы уверены, что хотите удалить коллекцию?"
+                class="btn btn-outline-danger"
+            >
+                Удалить
+            </button>
+        </form>
+    </div>
 </div>

app/blueprints/api/collections/templates/collections/collections_section.j2

@@ -0,0 +1,2 @@
+{% include "collections/create_form.j2" %}
+{% include "collections/select.j2" %}

app/blueprints/api/collections/templates/collections/create_form.j2

@@ -0,0 +1,33 @@
+<div id="collection-create-form" class="input-group has-validation">
+    <input
+        type="text"
+
+        {% if collections_error is defined %}
+            class="form-control is-invalid"
+        {% else %}
+            class="form-control"
+        {% endif %}
+
+        name="collection_name"
+        id="collection-name-input"
+        placeholder="Название базы"
+        aria-describedby="collection-feedback"
+        required
+    >
+    <button
+        type="submit"
+        class="btn btn-outline-primary"
+        hx-post="/api/collections"
+        hx-swap="outerHTML"
+        hx-target="#collection-create-form"
+        hx-indicator="#indicator"
+    >
+        Добавить
+    </button>
+
+    {% if collections_error is defined %}
+        <div id="collection-feedback" class="invalid-feedback">
+            {{ collections_error }}
+        </div>
+    {% endif %}
+</div>

app/blueprints/api/collections/templates/collections/grid.j2

@@ -1,8 +1,6 @@
 <div id="cards-grid" class="row row-cols-1 row-cols-sm-2 row-cols-md-3 row-cols-lg-4">
 {% for collection in collections %}
-    <div class="col mb-3">
-        {% include "collections/card.j2" %}
-    </div>
+    {% include "collections/card.j2" %}
 {% else %}
     <small class="form-text">Здесь ничего нет.</small>
 {% endfor %}

app/blueprints/api/collections/templates/collections/select.j2

@@ -0,0 +1,7 @@
+<select name="collection" id="collection-select" class="form-select flex-fill" size="3">
+    {% for collection in collections %}
+        <option value="{{collection.id}}">{{ collection.name }}</option>
+    {% else %}
+        <option disabled>Добавьте новую базу пользователей</option>
+    {% endfor %}
+</select>

app/blueprints/api/tasks/routes.py

@@ -1,11 +1,13 @@
 import uuid
 from flask import Blueprint, render_template, request
+from sqlalchemy import delete
 
 from app.extensions import db
+from app.models.collection import Collection
 from app.models.task import Task
 from app.models.session import Session
 
-from .tasks import send_messages_task, add_to_group_task
+from .tasks import parse_users_task, send_messages_task, add_to_group_task
 
 tasks = Blueprint("tasks", __name__, url_prefix="/tasks", template_folder="templates")
 
@@ -16,21 +18,27 @@ def get_tasks():
 
     return render_template("tasks_cards.j2", tasks=tasks, sessions=sessions)
 
-@tasks.route("/<int:task_id>", methods=["GET"])
-def get_task(task_id: int):
-    ...
-
 @tasks.route("/session/<int:session_id>", methods=["POST"])
 def create_task(session_id: int):
     info = request.form
+    collection_id = int(info['collection'])
+    
+    collection = Collection.query.get_or_404(collection_id)
     session = Session.query.get_or_404(session_id)
-    task =  Task(
-        name = str(uuid.uuid4()),
+
+    types = list(filter(lambda item: item in ('parse', 'add', 'message'), info.keys()))
+    
+    if len(types) > 0:
+        type = types[0]
+    
+    task = Task(
+        name = info.get('task_name') or str(uuid.uuid4()),
         session = session,
+        collection = collection,
         status = "CREATED",
         status_message = "Задача создана",
-        type = info.get('task'),
-        url = info.get('url'),
+        type = type,
+        url = info.get('parse') or info.get('add'),
         message = info.get('message'),
         file = None
     )
@@ -40,7 +48,7 @@ def create_task(session_id: int):
 
     start_task(task.id)
 
-    return 'Created', 200
+    return '', 204
 
 @tasks.route("/<int:task_id>", methods=["DELETE"])
 def delete_task(task_id: int):
@@ -49,7 +57,7 @@ def delete_task(task_id: int):
     db.session.delete(task)
     db.session.commit()
     
-    return 'Deleted', 200
+    return '', 204
 
 @tasks.route("/<int:task_id>/stop", methods=["PUT"])
 def stop_task(task_id: int):
@@ -72,7 +80,7 @@ def stop_task(task_id: int):
 
     db.session.commit()
     
-    return 'Stoping', 200
+    return '', 204
 
 
 @tasks.route("/<int:task_id>/start", methods=["PUT"])
@@ -85,8 +93,12 @@ def start_task(task_id: int):
 
         case 'add':
             add_to_group_task.delay(task_id=task_id)
+        
+        case 'parse':
+            parse_users_task.apply(task_id=task_id)
+            delete_task(task_id)
 
-    return 'Started', 200
+    return '', 204
 
 @tasks.route("/<int:task_id>/change", methods=["PUT"])
 def change_settings(task_id: int):
@@ -101,4 +113,4 @@ def change_settings(task_id: int):
 
     db.session.commit()
 
-    return 'Changed', 200
+    return '', 204

app/blueprints/api/tasks/tasks.py

@@ -2,6 +2,8 @@ import time
 
 from celery import shared_task
 import asyncio
+from app.models.session import Session
+from app.models.user import User
 
 from paper.parser import PaperParser
 from app.models.task import Task
@@ -10,14 +12,18 @@ from .handlers import run_state, failure_state, success_state
 
 async def add_to_group(session, task, task_self):
     async with PaperParser(session.name) as parser:
-        await parser.invite_users(session.users, task.url, task_self)
+        await parser.invite_users(task.collection.users, task.url, task_self)
 
 async def sending_message(session, task, task_self):
     async with PaperParser(session.name) as parser:
-        await parser.send_messages(session.users, task.message, task.file, task_self)
+        await parser.send_messages(task.collection.users, task.message, task.file, task_self)
+
+async def parse_users(session, task):
+    async with PaperParser(session.name) as parser:
+        return await parser.get_participants(task.url)
 
 @shared_task(bind=True)
-def add_to_group_task(self, task_id):
+def add_to_group_task(self, task_id: int):
     try:
         task: Task = Task.query.get(task_id)
         run_state(self.request.id, task_id)
@@ -37,10 +43,35 @@ def add_to_group_task(self, task_id):
     else:
         success_state(task_id)
 
+@shared_task(bind=True)
+def parse_users_task(self, task_id: int):
+    task: Task = Task.query.get_or_404(task_id)
+    session = task.session
+    collection = task.collection
 
+    users = asyncio.run(
+        parse_users(session, task)
+    )
+    
+    for user in users:
+        if not user.username:
+            continue
+        
+        if not User.exist(user.username, collection):
+            db.session.add(
+                User(
+                    first_name=user.first_name,
+                    last_name=user.last_name,
+                    username=user.username,
+                    phone=user.phone,
+                    collection=collection,
+                )
+            )
+    
+    db.session.commit()
 
 @shared_task(bind=True)
-def send_messages_task(self, task_id):
+def send_messages_task(self, task_id: int):
     task: Task = Task.query.get(task_id)
 
     session = task.session

app/blueprints/frontend/routes.py

@@ -1,6 +1,5 @@
 from flask import Blueprint, render_template
 from app.blueprints.api.sessions.routes import get_sessions
-from app.blueprints.api.users.routes import get_users
 from app.blueprints.api.tasks.routes import get_tasks
 from app.models.collection import Collection
 
@@ -21,7 +20,8 @@ def tasks():
 
 @frontend.route("/parse/<int:id>")
 def parse(id: int):
-    return render_template("parse.j2", session_id=id, users_template=get_users(id))
+    collections = Collection.query.all()
+    return render_template("parse.j2", session_id=id, collections=collections)
 
 @frontend.route("/collections")
 def collections():

app/blueprints/frontend/templates/parse.j2

@@ -5,12 +5,19 @@
 {% endblock title %}
 
 {% block main %}
-<form method="post" action="/test" class="container">
+<form method="post" action="/api/tasks/session/{{ session_id }}" class="container">
     <div class="row">
         <!-- Actions -->
         <section class="col-lg mb-3">
             <div class="separator">
-                <h2>Действия</h2>
+                <div class="d-flex gap-2">
+                    <h2>Действия</h2>
+                    <div id="indicator" class="htmx-indicator">
+                        <div class="spinner-border" role="status">
+                            <span class="visually-hidden">Loading...</span>
+                        </div>
+                    </div>
+                </div>
                 <hr class="divider">
             </div>
             <nav class="mb-3">
@@ -24,14 +31,14 @@
                 
                 <section class="tab-pane fade show active" id="actions-parse-section">
                         <div class="form-floating">
-                            <input type="text" class="form-control" name="group" id="group-from-input" placeholder="Группа для парсинга">
+                            <input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга">
                             <label for="group-from-input">Группа для парсинга</label>
                         </div>
                 </section>
                 
                 <section class="tab-pane fade" id="actions-add-section">
                     <div class="form-floating">
-                        <input type="text" name="url" class="form-control" id="group-to-input"
+                        <input type="text" name="url" name="add" class="form-control" id="group-to-input"
                             placeholder="Группа для добавлнеия">
                         <label for="group-to-input">Группа для добавления</label>
                     </div>
@@ -63,7 +70,7 @@
                 <input
                     type="text"
                     class="form-control"
-                    name="collection"
+                    name="task_name"
                     id="task-name-input"
                     placeholder="Название задачи"
                 >
@@ -71,29 +78,9 @@
             </div>
             
             <section class="d-flex flex-column gap-3 mb-3">
-                <div class="input-group">
-                    <div class="form-floating">
-                        <input
-                            type="text"
-                            class="form-control"
-                            name="collection"
-                            id="collection-name-input"
-                            placeholder="Группа для парсинга"
-                        >
-                        <label for="collection-name-input">Название базы пользователей</label>
-                    </div>
-                    <button type="submit" class="btn btn-outline-primary">Добавить</button>
-                </div>
-
-                <select class="form-select flex-fill" size="3">
-                    {% for collection in collections %}
-                        <option value="{{collection.id}}">{{ collection.name }}</option>
-                    {% else %}
-                        <option disabled>Добавьте новую базу пользователей</option>
-                    {% endfor %}
-                </select>
+                {% include "collections/collections_section.j2" %}
             </section>
-            <button type="submit" class="w-100 btn btn-primary">Создать задачу</button>
+            <button type="submit" hx-post="/api/tasks/session/{{ session_id }}" hx-indicator="#indicator" class="w-100 btn btn-primary">Создать задачу</button>
         </section>
     </div>
 </form>

app/models/collection.py

@@ -9,4 +9,9 @@ class Collection(db.Model):
     name: Mapped[str] = mapped_column(unique=True, nullable=False)
     
     tasks: Mapped[List['Task']] = relationship("Task", back_populates="collection")
-    users: Mapped[List['User']] = relationship("User", cascade="all, delete-orphan", back_populates="collection")
+    users: Mapped[List['User']] = relationship("User", cascade="all, delete-orphan", back_populates="collection")
+    
+    def exists(name: str):
+        collections = Collection.query.filter_by(name=name).all()
+        
+        return len(collections) > 0

app/models/user.py

@@ -16,4 +16,7 @@ class User(db.Model):
     username: Mapped[str] = mapped_column(nullable=True, unique=True)
     
     collection_id = Column(Integer, ForeignKey("collection.id"))
-    collection: Mapped["Collection"] = relationship("Collection", back_populates="users") 
+    collection: Mapped["Collection"] = relationship("Collection", back_populates="users")
+    
+    def exist(username: str, collection):
+        return not User.query.filter_by(username=username, collection=collection).first() is None

paper/parser.py

@@ -1,11 +1,9 @@
 import asyncio
-import os
 
 from loguru import logger
 
 from paper.client import PaperClient
 from paper.errors import IgnoreException, NeedPasswordException, UserPrivacyException
-from paper.models import Message
 
 
 class PaperParser:
@@ -17,7 +15,7 @@ class PaperParser:
 
         self.users_to_delete = []
 
-    async def invite_users(self, users, group, task = None):
+    async def invite_users(self, users, group, task):
         await self.client.invite_self(group)
 
         group_participants = await self.client.get_participants(group)
@@ -33,7 +31,7 @@ class PaperParser:
 
                 await self.client.invite_user(user, group)
 
-            except (UserPrivacyException, IgnoreException) as e:
+            except (UserPrivacyException, IgnoreException):
                 self.users_to_delete.append(user)
                 logger.warning("Exception occurred. Skipping user...")
             
@@ -47,6 +45,8 @@ class PaperParser:
         #     if dialog.is_user:
         #         messages = tuple(filter(lambda chat_message: message.text == chat_message.text, await self.client.get_messages(dialog)))
         #         print(messages)
+        if not task:
+            return
 
         for user in users:
             try:
@@ -55,7 +55,7 @@ class PaperParser:
                 
                 await self.client.send_message(user, message, file)
 
-            except (UserPrivacyException, IgnoreException) as e:
+            except (UserPrivacyException, IgnoreException):
                 self.users_to_delete.append(user)
                 logger.warning("Exception occurred. Skipping user...")
 
@@ -84,4 +84,4 @@ class PaperParser:
         return self
 
     async def __aexit__(self, *args, **kwargs):
-        await self.client.disconnect()
+        await self.client.disconnect()