Fixes
This commit is contained in:
parent
da4dddbcf7
commit
111a81ae26
|
@ -23,6 +23,9 @@ def create_task(session_id: int):
|
||||||
info = request.form
|
info = request.form
|
||||||
collection_id = int(info['collection'])
|
collection_id = int(info['collection'])
|
||||||
|
|
||||||
|
#! FIXME: При первой же возможности добавить метаданные в задачи
|
||||||
|
parse_bio = True if info.get('parse-bio-check') == "on" else False
|
||||||
|
|
||||||
collection = Collection.query.get_or_404(collection_id)
|
collection = Collection.query.get_or_404(collection_id)
|
||||||
session = Session.query.get_or_404(session_id)
|
session = Session.query.get_or_404(session_id)
|
||||||
|
|
||||||
|
@ -42,7 +45,7 @@ def create_task(session_id: int):
|
||||||
type = type,
|
type = type,
|
||||||
url = info.get('parse') or info.get('add'),
|
url = info.get('parse') or info.get('add'),
|
||||||
message = info.get('message'),
|
message = info.get('message'),
|
||||||
file = None
|
file = None or str(parse_bio)
|
||||||
)
|
)
|
||||||
|
|
||||||
db.session.add(task)
|
db.session.add(task)
|
||||||
|
@ -97,10 +100,7 @@ def start_task(task_id: int):
|
||||||
add_to_group_task.delay(task_id=task_id)
|
add_to_group_task.delay(task_id=task_id)
|
||||||
|
|
||||||
case 'parse':
|
case 'parse':
|
||||||
try:
|
parse_users_task.delay(task_id=task_id)
|
||||||
parse_users_task.run(task_id=task_id)
|
|
||||||
finally:
|
|
||||||
delete_task(task_id)
|
|
||||||
|
|
||||||
return '', 204
|
return '', 204
|
||||||
|
|
||||||
|
|
|
@ -20,16 +20,25 @@ async def sending_message(session, task, task_self):
|
||||||
async with PaperParser(session.name) as parser:
|
async with PaperParser(session.name) as parser:
|
||||||
await parser.send_messages(task.collection.users, task.message, task.file, task_self)
|
await parser.send_messages(task.collection.users, task.message, task.file, task_self)
|
||||||
|
|
||||||
async def parse_users(session, task, db, collection):
|
async def parse_users(session, task, db, collection, task_self):
|
||||||
|
parse_bio = True if task.file else False
|
||||||
|
|
||||||
async with PaperParser(session.name) as parser:
|
async with PaperParser(session.name) as parser:
|
||||||
users = await parser.get_participants(task.url)
|
users = await parser.get_participants(task.url)
|
||||||
|
|
||||||
filtered_users = filter(filter_users, users)
|
filtered_users = filter(filter_users, users)
|
||||||
|
|
||||||
for user in filtered_users:
|
for user in filtered_users:
|
||||||
|
if task_self.is_aborted():
|
||||||
|
return
|
||||||
|
|
||||||
if not User.exist(user.username, collection):
|
if not User.exist(user.username, collection):
|
||||||
full = await parser.get_full_info()
|
bio = None
|
||||||
|
|
||||||
|
if parse_bio:
|
||||||
|
full = await parser.get_full_info(user)
|
||||||
bio = full.full_user.about if full.full_user else None
|
bio = full.full_user.about if full.full_user else None
|
||||||
|
logger.info(f"Got bio for {user.username}. Bio: { bio }")
|
||||||
|
|
||||||
db.session.add(
|
db.session.add(
|
||||||
User(
|
User(
|
||||||
|
@ -44,6 +53,8 @@ async def parse_users(session, task, db, collection):
|
||||||
|
|
||||||
db.session.commit()
|
db.session.commit()
|
||||||
|
|
||||||
|
await asyncio.sleep(5)
|
||||||
|
|
||||||
def filter_users(user):
|
def filter_users(user):
|
||||||
if not user.username or user.bot:
|
if not user.username or user.bot:
|
||||||
return False
|
return False
|
||||||
|
@ -89,7 +100,7 @@ def parse_users_task(self, task_id: int):
|
||||||
run_state(self.request.id, task_id)
|
run_state(self.request.id, task_id)
|
||||||
|
|
||||||
asyncio.run(
|
asyncio.run(
|
||||||
parse_users(session, task, db, collection)
|
parse_users(session, task, db, collection, self)
|
||||||
)
|
)
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
@ -30,10 +30,16 @@
|
||||||
<div class="tab-content" id="nav-tabContent">
|
<div class="tab-content" id="nav-tabContent">
|
||||||
|
|
||||||
<section class="tab-pane fade show active" id="actions-parse-section">
|
<section class="tab-pane fade show active" id="actions-parse-section">
|
||||||
<div class="form-floating">
|
<div class="form-floating mb-3">
|
||||||
<input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга">
|
<input type="text" class="form-control" name="parse" id="group-from-input" placeholder="Группа для парсинга">
|
||||||
<label for="group-from-input">Группа для парсинга</label>
|
<label for="group-from-input">Группа для парсинга</label>
|
||||||
</div>
|
</div>
|
||||||
|
<div class="form-check">
|
||||||
|
<input class="form-check-input" type="checkbox" name="parse-bio-check" id="parse-bio-check">
|
||||||
|
<label class="form-check-label" for="parse-bio-check">
|
||||||
|
Парсить с описанием?
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<section class="tab-pane fade" id="actions-add-section">
|
<section class="tab-pane fade" id="actions-add-section">
|
||||||
|
|
|
@ -16,7 +16,7 @@ class User(db.Model):
|
||||||
username: Mapped[str] = mapped_column(nullable=True)
|
username: Mapped[str] = mapped_column(nullable=True)
|
||||||
description: Mapped[str] = mapped_column(nullable=True)
|
description: Mapped[str] = mapped_column(nullable=True)
|
||||||
|
|
||||||
delete: Mapped[bool] = mapped_column(default=false)
|
delete: Mapped[bool] = mapped_column(default=False)
|
||||||
|
|
||||||
collection_id = Column(Integer, ForeignKey("collection.id"))
|
collection_id = Column(Integer, ForeignKey("collection.id"))
|
||||||
collection: Mapped["Collection"] = relationship("Collection", back_populates="users")
|
collection: Mapped["Collection"] = relationship("Collection", back_populates="users")
|
||||||
|
|
Loading…
Reference in New Issue