Compare commits

..

No commits in common. "28242afac822028daf990eeb3698f2bca05bf513" and "da4dddbcf709998d20c5e645b766d3b3983d18c9" have entirely different histories.

4 changed files with 10 additions and 30 deletions

View File

@ -23,9 +23,6 @@ def create_task(session_id: int):
info = request.form info = request.form
collection_id = int(info['collection']) collection_id = int(info['collection'])
#! FIXME: При первой же возможности добавить метаданные в задачи
parse_bio = True if info.get('parse-bio-check') == "on" else False
collection = Collection.query.get_or_404(collection_id) collection = Collection.query.get_or_404(collection_id)
session = Session.query.get_or_404(session_id) session = Session.query.get_or_404(session_id)
@ -45,7 +42,7 @@ def create_task(session_id: int):
type = type, type = type,
url = info.get('parse') or info.get('add'), url = info.get('parse') or info.get('add'),
message = info.get('message'), message = info.get('message'),
file = None or str(parse_bio) file = None
) )
db.session.add(task) db.session.add(task)
@ -100,10 +97,10 @@ def start_task(task_id: int):
add_to_group_task.delay(task_id=task_id) add_to_group_task.delay(task_id=task_id)
case 'parse': case 'parse':
if task.file: try:
parse_users_task.delay(task_id=task_id)
else:
parse_users_task.run(task_id=task_id) parse_users_task.run(task_id=task_id)
finally:
delete_task(task_id)
return '', 204 return '', 204

View File

@ -20,25 +20,16 @@ async def sending_message(session, task, task_self):
async with PaperParser(session.name) as parser: async with PaperParser(session.name) as parser:
await parser.send_messages(task.collection.users, task.message, task.file, task_self) await parser.send_messages(task.collection.users, task.message, task.file, task_self)
async def parse_users(session, task, db, collection, task_self): async def parse_users(session, task, db, collection):
parse_bio = True if task.file else False
async with PaperParser(session.name) as parser: async with PaperParser(session.name) as parser:
users = await parser.get_participants(task.url) users = await parser.get_participants(task.url)
filtered_users = filter(filter_users, users) filtered_users = filter(filter_users, users)
for user in filtered_users: for user in filtered_users:
if task_self.is_aborted():
return
if not User.exist(user.username, collection): if not User.exist(user.username, collection):
bio = None full = await parser.get_full_info()
bio = full.full_user.about if full.full_user else None
if parse_bio:
full = await parser.get_full_info(user)
bio = full.full_user.about if full.full_user else None
logger.info(f"Got bio for {user.username}. Bio: { bio }")
db.session.add( db.session.add(
User( User(
@ -53,8 +44,6 @@ async def parse_users(session, task, db, collection, task_self):
db.session.commit() db.session.commit()
await asyncio.sleep(5)
def filter_users(user): def filter_users(user):
if not user.username or user.bot: if not user.username or user.bot:
return False return False
@ -100,7 +89,7 @@ def parse_users_task(self, task_id: int):
run_state(self.request.id, task_id) run_state(self.request.id, task_id)
asyncio.run( asyncio.run(
parse_users(session, task, db, collection, self) parse_users(session, task, db, collection)
) )
except Exception as e: except Exception as e:

View File

@ -30,16 +30,10 @@
<div class="tab-content" id="nav-tabContent"> <div class="tab-content" id="nav-tabContent">
<section class="tab-pane fade show active" id="actions-parse-section"> <section class="tab-pane fade show active" id="actions-parse-section">
<div class="form-floating mb-3"> <div class="form-floating">
<input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга"> <input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга">
<label for="group-from-input">Группа для парсинга</label> <label for="group-from-input">Группа для парсинга</label>
</div> </div>
<div class="form-check">
<input class="form-check-input" type="checkbox" name="parse-bio-check" id="parse-bio-check">
<label class="form-check-label" for="parse-bio-check">
Парсить с описанием?
</label>
</div>
</section> </section>
<section class="tab-pane fade" id="actions-add-section"> <section class="tab-pane fade" id="actions-add-section">

View File

@ -16,7 +16,7 @@ class User(db.Model):
username: Mapped[str] = mapped_column(nullable=True) username: Mapped[str] = mapped_column(nullable=True)
description: Mapped[str] = mapped_column(nullable=True) description: Mapped[str] = mapped_column(nullable=True)
delete: Mapped[bool] = mapped_column(default=False) delete: Mapped[bool] = mapped_column(default=false)
collection_id = Column(Integer, ForeignKey("collection.id")) collection_id = Column(Integer, ForeignKey("collection.id"))
collection: Mapped["Collection"] = relationship("Collection", back_populates="users") collection: Mapped["Collection"] = relationship("Collection", back_populates="users")