This commit is contained in:
Анатолий Богомолов 2024-04-12 21:41:37 +10:00
parent da4dddbcf7
commit 111a81ae26
4 changed files with 28 additions and 11 deletions

View File

@ -23,6 +23,9 @@ def create_task(session_id: int):
info = request.form info = request.form
collection_id = int(info['collection']) collection_id = int(info['collection'])
#! FIXME: При первой же возможности добавить метаданные в задачи
parse_bio = True if info.get('parse-bio-check') == "on" else False
collection = Collection.query.get_or_404(collection_id) collection = Collection.query.get_or_404(collection_id)
session = Session.query.get_or_404(session_id) session = Session.query.get_or_404(session_id)
@ -42,7 +45,7 @@ def create_task(session_id: int):
type = type, type = type,
url = info.get('parse') or info.get('add'), url = info.get('parse') or info.get('add'),
message = info.get('message'), message = info.get('message'),
file = None file = None or str(parse_bio)
) )
db.session.add(task) db.session.add(task)
@ -97,10 +100,7 @@ def start_task(task_id: int):
add_to_group_task.delay(task_id=task_id) add_to_group_task.delay(task_id=task_id)
case 'parse': case 'parse':
try: parse_users_task.delay(task_id=task_id)
parse_users_task.run(task_id=task_id)
finally:
delete_task(task_id)
return '', 204 return '', 204

View File

@ -20,16 +20,25 @@ async def sending_message(session, task, task_self):
async with PaperParser(session.name) as parser: async with PaperParser(session.name) as parser:
await parser.send_messages(task.collection.users, task.message, task.file, task_self) await parser.send_messages(task.collection.users, task.message, task.file, task_self)
async def parse_users(session, task, db, collection): async def parse_users(session, task, db, collection, task_self):
parse_bio = True if task.file else False
async with PaperParser(session.name) as parser: async with PaperParser(session.name) as parser:
users = await parser.get_participants(task.url) users = await parser.get_participants(task.url)
filtered_users = filter(filter_users, users) filtered_users = filter(filter_users, users)
for user in filtered_users: for user in filtered_users:
if task_self.is_aborted():
return
if not User.exist(user.username, collection): if not User.exist(user.username, collection):
full = await parser.get_full_info() bio = None
bio = full.full_user.about if full.full_user else None
if parse_bio:
full = await parser.get_full_info(user)
bio = full.full_user.about if full.full_user else None
logger.info(f"Got bio for {user.username}. Bio: { bio }")
db.session.add( db.session.add(
User( User(
@ -44,6 +53,8 @@ async def parse_users(session, task, db, collection):
db.session.commit() db.session.commit()
await asyncio.sleep(5)
def filter_users(user): def filter_users(user):
if not user.username or user.bot: if not user.username or user.bot:
return False return False
@ -89,7 +100,7 @@ def parse_users_task(self, task_id: int):
run_state(self.request.id, task_id) run_state(self.request.id, task_id)
asyncio.run( asyncio.run(
parse_users(session, task, db, collection) parse_users(session, task, db, collection, self)
) )
except Exception as e: except Exception as e:

View File

@ -30,10 +30,16 @@
<div class="tab-content" id="nav-tabContent"> <div class="tab-content" id="nav-tabContent">
<section class="tab-pane fade show active" id="actions-parse-section"> <section class="tab-pane fade show active" id="actions-parse-section">
<div class="form-floating"> <div class="form-floating mb-3">
<input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга"> <input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга">
<label for="group-from-input">Группа для парсинга</label> <label for="group-from-input">Группа для парсинга</label>
</div> </div>
<div class="form-check">
<input class="form-check-input" type="checkbox" name="parse-bio-check" id="parse-bio-check">
<label class="form-check-label" for="parse-bio-check">
Парсить с описанием?
</label>
</div>
</section> </section>
<section class="tab-pane fade" id="actions-add-section"> <section class="tab-pane fade" id="actions-add-section">

View File

@ -16,7 +16,7 @@ class User(db.Model):
username: Mapped[str] = mapped_column(nullable=True) username: Mapped[str] = mapped_column(nullable=True)
description: Mapped[str] = mapped_column(nullable=True) description: Mapped[str] = mapped_column(nullable=True)
delete: Mapped[bool] = mapped_column(default=false) delete: Mapped[bool] = mapped_column(default=False)
collection_id = Column(Integer, ForeignKey("collection.id")) collection_id = Column(Integer, ForeignKey("collection.id"))
collection: Mapped["Collection"] = relationship("Collection", back_populates="users") collection: Mapped["Collection"] = relationship("Collection", back_populates="users")